blazer 2.6.5 → 3.0.0

data/app/controllers/blazer/dashboards_controller.rb

@@ -65,35 +65,35 @@ module Blazer
 
     private
 
-      def dashboard_params
-        params.require(:dashboard).permit(:name)
-      end
+    def dashboard_params
+      params.require(:dashboard).permit(:name)
+    end
 
-      def set_dashboard
-        @dashboard = Blazer::Dashboard.find(params[:id])
-      end
+    def set_dashboard
+      @dashboard = Blazer::Dashboard.find(params[:id])
+    end
 
-      def update_dashboard(dashboard)
-        dashboard.assign_attributes(dashboard_params)
-        Blazer::Dashboard.transaction do
-          if params[:query_ids].is_a?(Array)
-            query_ids = params[:query_ids].map(&:to_i)
-            @queries = Blazer::Query.find(query_ids).sort_by { |q| query_ids.index(q.id) }
-          end
-          if dashboard.save
-            if @queries
-              @queries.each_with_index do |query, i|
-                dashboard_query = dashboard.dashboard_queries.where(query_id: query.id).first_or_initialize
-                dashboard_query.position = i
-                dashboard_query.save!
-              end
-              if dashboard.persisted?
-                dashboard.dashboard_queries.where.not(query_id: query_ids).destroy_all
-              end
+    def update_dashboard(dashboard)
+      dashboard.assign_attributes(dashboard_params)
+      Blazer::Dashboard.transaction do
+        if params[:query_ids].is_a?(Array)
+          query_ids = params[:query_ids].map(&:to_i)
+          @queries = Blazer::Query.find(query_ids).sort_by { |q| query_ids.index(q.id) }
+        end
+        if dashboard.save
+          if @queries
+            @queries.each_with_index do |query, i|
+              dashboard_query = dashboard.dashboard_queries.where(query_id: query.id).first_or_initialize
+              dashboard_query.position = i
+              dashboard_query.save!
+            end
+            if dashboard.persisted?
+              dashboard.dashboard_queries.where.not(query_id: query_ids).destroy_all
             end
-            true
           end
+          true
         end
       end
+    end
   end
 end

data/app/controllers/blazer/queries_controller.rb

@@ -73,6 +73,12 @@ module Blazer
       @query.update!(status: "active") if @query.respond_to?(:status) && @query.status.in?(["archived", nil])
 
       add_cohort_analysis_vars if @query.cohort_analysis?
+
+      if @success
+        @run_data = {statement: @query.statement, query_id: @query.id, data_source: @query.data_source, variables: variable_params(@query)}
+        @run_data[:forecast] = "t" if params[:forecast]
+        @run_data[:cohort_period] = params[:cohort_period] if params[:cohort_period]
+      end
     end
 
     def edit
@@ -82,7 +88,6 @@ module Blazer
       @query = Query.find_by(id: params[:query_id]) if params[:query_id]
 
       # use query data source when present
-      # need to update viewable? logic below if this changes
       data_source = @query.data_source if @query && @query.data_source
       data_source ||= params[:data_source]
       @data_source = Blazer.data_sources[data_source]
@@ -100,10 +105,9 @@ module Blazer
 
       run_cohort_analysis if @cohort_analysis
 
-      # ensure viewable
-      if !(@query || Query.new(data_source: @data_source.id)).viewable?(blazer_user)
-        render_forbidden
-      elsif @run_id
+      query_running = !@run_id.nil?
+
+      if query_running
         @timestamp = blazer_params[:timestamp].to_i
 
         @result = @data_source.run_results(@run_id)
@@ -131,8 +135,10 @@ module Blazer
       elsif @success
         @run_id = blazer_run_id
 
-        options = {user: blazer_user, query: @query, refresh_cache: params[:check], run_id: @run_id, async: Blazer.async}
-        if Blazer.async && request.format.symbol != :csv
+        async = Blazer.async
+
+        options = {user: blazer_user, query: @query, refresh_cache: params[:check], run_id: @run_id, async: async}
+        if async && request.format.symbol != :csv
           Blazer::RunStatementJob.perform_later(@data_source.id, @statement.statement, options.merge(values: @statement.values))
           wait_start = Blazer.monotonic_time
           loop do
@@ -145,7 +151,7 @@ module Blazer
         end
 
         if @result
-          @data_source.delete_results(@run_id) if @run_id
+          @data_source.delete_results(@run_id) if @run_id && async
 
           @columns = @result.columns
           @rows = @result.rows
@@ -217,229 +223,245 @@ module Blazer
 
     private
 
-      def set_data_source
-        @data_source = Blazer.data_sources[params[:data_source]]
+    def set_data_source
+      @data_source = Blazer.data_sources[params[:data_source]]
+    end
 
-        unless Query.new(data_source: @data_source.id).editable?(blazer_user)
-          render_forbidden
+    def continue_run
+      render json: {run_id: @run_id, timestamp: @timestamp}, status: :accepted
+    end
+
+    def render_run
+      @checks = @query ? @query.checks.order(:id) : []
+
+      @first_row = @rows.first || []
+      @column_types = []
+      if @rows.any?
+        @columns.each_with_index do |_, i|
+          @column_types << (
+            case @first_row[i]
+            when Integer
+              "int"
+            when Float, BigDecimal
+              "float"
+            else
+              "string-ins"
+            end
+          )
         end
       end
 
-      def continue_run
-        render json: {run_id: @run_id, timestamp: @timestamp}, status: :accepted
-      end
+      @min_width_types = @columns.each_with_index.select { |c, i| @first_row[i].is_a?(Time) || @first_row[i].is_a?(String) || @data_source.smart_columns[c] }.map(&:last)
 
-      def render_run
-        @checks = @query ? @query.checks.order(:id) : []
-
-        @first_row = @rows.first || []
-        @column_types = []
-        if @rows.any?
-          @columns.each_with_index do |_, i|
-            @column_types << (
-              case @first_row[i]
-              when Integer
-                "int"
-              when Float, BigDecimal
-                "float"
-              else
-                "string-ins"
-              end
-            )
-          end
-        end
+      @smart_values = @result.smart_values if @result
 
-        @min_width_types = @columns.each_with_index.select { |c, i| @first_row[i].is_a?(Time) || @first_row[i].is_a?(String) || @data_source.smart_columns[c] }.map(&:last)
-
-        @boom = @result.boom if @result
-
-        @linked_columns = @data_source.linked_columns
-
-        @markers = []
-        [["latitude", "longitude"], ["lat", "lon"], ["lat", "lng"]].each do |keys|
-          lat_index = @columns.index(keys.first)
-          lon_index = @columns.index(keys.last)
-          if lat_index && lon_index
-            @markers =
-              @rows.select do |r|
-                r[lat_index] && r[lon_index]
-              end.map do |r|
-                {
-                  # Mapbox.js does sanitization with https://github.com/mapbox/sanitize-caja
-                  # but we should do it here as well
-                  title: r.each_with_index.map { |v, i| i == lat_index || i == lon_index ? nil : "<strong>#{ERB::Util.html_escape(@columns[i])}:</strong> #{ERB::Util.html_escape(v)}" }.compact.join("<br />").truncate(140),
-                  latitude: r[lat_index],
-                  longitude: r[lon_index]
-                }
-              end
-          end
-        end
+      @linked_columns = @data_source.linked_columns
 
-        render_cohort_analysis if @cohort_analysis && !@error
+      @markers = []
+      @geojson = []
+      set_map_data if Blazer.maps?
 
-        respond_to do |format|
-          format.html do
-            render layout: false
-          end
-          format.csv do
-            # not ideal, but useful for testing
-            raise Error, @error if @error && Rails.env.test?
+      render_cohort_analysis if @cohort_analysis && !@error
 
-            send_data csv_data(@columns, @rows, @data_source), type: "text/csv; charset=utf-8; header=present", disposition: "attachment; filename=\"#{@query.try(:name).try(:parameterize).presence || 'query'}.csv\""
-          end
+      respond_to do |format|
+        format.html do
+          render layout: false
         end
-      end
-
-      def set_queries(limit = nil)
-        @queries = Blazer::Query.named.select(:id, :name, :creator_id, :statement)
-        @queries = @queries.includes(:creator) if Blazer.user_class
+        format.csv do
+          # not ideal, but useful for testing
+          raise Error, @error if @error && Rails.env.test?
 
-        if blazer_user && params[:filter] == "mine"
-          @queries = @queries.where(creator_id: blazer_user.id).reorder(updated_at: :desc)
-        elsif blazer_user && params[:filter] == "viewed" && Blazer.audit
-          @queries = queries_by_ids(Blazer::Audit.where(user_id: blazer_user.id).order(created_at: :desc).limit(500).pluck(:query_id).uniq)
-        else
-          @queries = @queries.limit(limit) if limit
-          @queries = @queries.active.order(:name)
+          data = csv_data(@columns, @rows, @data_source)
+          filename = "#{@query.try(:name).try(:parameterize).presence || 'query'}.csv"
+          send_data data, type: "text/csv; charset=utf-8", disposition: "attachment", filename: filename
         end
-        @queries = @queries.to_a
+      end
+    end
 
-        @more = limit && @queries.size >= limit
+    def set_map_data
+      [["latitude", "longitude"], ["lat", "lon"], ["lat", "lng"]].each do |keys|
+        lat_index = @columns.index(keys.first)
+        lon_index = @columns.index(keys.last)
+        if lat_index && lon_index
+          @markers =
+            @rows.select do |r|
+              r[lat_index] && r[lon_index]
+            end.map do |r|
+              {
+                tooltip: map_tooltip(r.each_with_index.reject { |v, i| i == lat_index || i == lon_index }),
+                latitude: r[lat_index],
+                longitude: r[lon_index]
+              }
+            end
 
-        @queries = @queries.select { |q| !q.name.to_s.start_with?("#") || q.try(:creator).try(:id) == blazer_user.try(:id) }
+          return if @markers.any?
+        end
+      end
 
-        @queries =
-          @queries.map do |q|
-            {
-              id: q.id,
-              name: q.name,
-              creator: blazer_user && q.try(:creator) == blazer_user ? "You" : q.try(:creator).try(Blazer.user_name),
-              vars: q.variables.join(", "),
-              to_param: q.to_param
-            }
+      geo_index = @columns.index("geojson")
+      if geo_index
+        @geojson =
+          @rows.filter_map do |r|
+            if r[geo_index].is_a?(String) && (geometry = (JSON.parse(r[geo_index]) rescue nil)) && geometry.is_a?(Hash)
+              {
+                tooltip: map_tooltip(r.each_with_index.reject { |v, i| i == geo_index }),
+                geometry: geometry
+              }
+            end
           end
       end
+    end
 
-      def queries_by_ids(favorite_query_ids)
-        queries = Blazer::Query.active.named.where(id: favorite_query_ids)
-        queries = queries.includes(:creator) if Blazer.user_class
-        queries = queries.index_by(&:id)
-        favorite_query_ids.map { |query_id| queries[query_id] }.compact
+    def map_tooltip(r)
+      r.map { |v, i| "<strong>#{ERB::Util.html_escape(@columns[i])}:</strong> #{ERB::Util.html_escape(v)}" }.join("<br>").truncate(140, separator: " ")
+    end
+
+    def set_queries(limit = nil)
+      @queries = Blazer::Query.named.select(:id, :name, :creator_id, :statement)
+      @queries = @queries.includes(:creator) if Blazer.user_class
+
+      if blazer_user && params[:filter] == "mine"
+        @queries = @queries.where(creator_id: blazer_user.id).reorder(updated_at: :desc)
+      elsif blazer_user && params[:filter] == "viewed" && Blazer.audit
+        @queries = queries_by_ids(Blazer::Audit.where(user_id: blazer_user.id).order(created_at: :desc).limit(500).pluck(:query_id).uniq)
+      else
+        @queries = @queries.limit(limit) if limit
+        @queries = @queries.active.order(:name)
       end
+      @queries = @queries.to_a
+
+      @more = limit && @queries.size >= limit
 
-      def set_query
-        @query = Blazer::Query.find(params[:id].to_s.split("-").first)
+      @queries = @queries.select { |q| !q.name.to_s.start_with?("#") || q.try(:creator).try(:id) == blazer_user.try(:id) }
 
-        unless @query.viewable?(blazer_user)
-          render_forbidden
+      @queries =
+        @queries.map do |q|
+          {
+            id: q.id,
+            name: q.name,
+            creator: blazer_user && q.try(:creator) == blazer_user ? "You" : q.try(:creator).try(Blazer.user_name),
+            vars: q.variables.join(", "),
+            to_param: q.to_param
+          }
         end
-      end
+    end
 
-      def render_forbidden
-        render plain: "Access denied", status: :forbidden
-      end
+    def queries_by_ids(favorite_query_ids)
+      queries = Blazer::Query.active.named.where(id: favorite_query_ids)
+      queries = queries.includes(:creator) if Blazer.user_class
+      queries = queries.index_by(&:id)
+      favorite_query_ids.map { |query_id| queries[query_id] }.compact
+    end
 
-      def query_params
-        params.require(:query).permit(:name, :description, :statement, :data_source)
-      end
+    def set_query
+      @query = Blazer::Query.find(params[:id].to_s.split("-").first)
+    end
 
-      def blazer_params
-        params[:blazer] || {}
-      end
+    def render_forbidden
+      render plain: "Access denied", status: :forbidden
+    end
 
-      def csv_data(columns, rows, data_source)
-        CSV.generate do |csv|
-          csv << columns
-          rows.each do |row|
-            csv << row.each_with_index.map { |v, i| v.is_a?(Time) ? blazer_time_value(data_source, columns[i], v) : v }
-          end
+    def query_params
+      params.require(:query).permit(:name, :description, :statement, :data_source)
+    end
+
+    def blazer_params
+      params[:blazer] || {}
+    end
+
+    def csv_data(columns, rows, data_source)
+      CSV.generate do |csv|
+        csv << columns
+        rows.each do |row|
+          csv << row.each_with_index.map { |v, i| v.is_a?(Time) ? blazer_time_value(data_source, columns[i], v) : v }
         end
       end
+    end
 
-      def blazer_time_value(data_source, k, v)
-        data_source.local_time_suffix.any? { |s| k.ends_with?(s) } ? v.to_s.sub(" UTC", "") : v.in_time_zone(Blazer.time_zone)
-      end
-      helper_method :blazer_time_value
+    def blazer_time_value(data_source, k, v)
+      data_source.local_time_suffix.any? { |s| k.ends_with?(s) } ? v.to_s.sub(" UTC", "") : v.in_time_zone(Blazer.time_zone)
+    end
+    helper_method :blazer_time_value
+
+    def blazer_run_id
+      params[:run_id].to_s.gsub(/[^a-z0-9\-]/i, "")
+    end
 
-      def blazer_run_id
-        params[:run_id].to_s.gsub(/[^a-z0-9\-]/i, "")
+    def run_cohort_analysis
+      unless @statement.data_source.supports_cohort_analysis?
+        @cohort_error = "This data source does not support cohort analysis"
       end
 
-      def run_cohort_analysis
-        unless @statement.data_source.supports_cohort_analysis?
-          @cohort_error = "This data source does not support cohort analysis"
-        end
+      @show_cohort_rows = !params[:query_id] || @cohort_error
+      cohort_analysis_statement(@statement) unless @show_cohort_rows
+    end
 
-        @show_cohort_rows = !params[:query_id] || @cohort_error
-        cohort_analysis_statement(@statement) unless @show_cohort_rows
-      end
+    def render_cohort_analysis
+      if @show_cohort_rows
+        @cohort_analysis = false
 
-      def render_cohort_analysis
-        if @show_cohort_rows
-          @cohort_analysis = false
+        @row_limit = 1000
 
-          @row_limit = 1000
+        # check results
+        unless @cohort_error
+          # check names
+          expected_columns = ["user_id", "conversion_time"]
+          missing_columns = expected_columns - @result.columns
+          if missing_columns.any?
+            @cohort_error = "Expected user_id and conversion_time columns"
+          end
 
-          # check results
+          # check types (user_id can be any type)
           unless @cohort_error
-            # check names
-            expected_columns = ["user_id", "conversion_time"]
-            missing_columns = expected_columns - @result.columns
-            if missing_columns.any?
-              @cohort_error = "Expected user_id and conversion_time columns"
-            end
-
-            # check types (user_id can be any type)
-            unless @cohort_error
-              column_types = @result.columns.zip(@result.column_types).to_h
+            column_types = @result.columns.zip(@result.column_types).to_h
 
-              if !column_types["cohort_time"].in?(["time", nil])
-                @cohort_error = "cohort_time must be time column"
-              elsif !column_types["conversion_time"].in?(["time", nil])
-                @cohort_error = "conversion_time must be time column"
-              end
+            if !column_types["cohort_time"].in?(["time", nil])
+              @cohort_error = "cohort_time must be time column"
+            elsif !column_types["conversion_time"].in?(["time", nil])
+              @cohort_error = "conversion_time must be time column"
             end
           end
-        else
-          @today = Blazer.time_zone.today
-          @min_cohort_date, @max_cohort_date = @result.rows.map { |r| r[0] }.minmax
-          @buckets = {}
-          @rows.each do |r|
-            @buckets[[r[0], r[1]]] = r[2]
-          end
-
-          @cohort_dates = []
-          current_date = @max_cohort_date
-          while current_date && current_date >= @min_cohort_date
-            @cohort_dates << current_date
-            current_date =
-              case @cohort_period
-              when "day"
-                current_date - 1
-              when "week"
-                current_date - 7
-              else
-                current_date.prev_month
-              end
-          end
+        end
+      else
+        @today = Blazer.time_zone.today
+        @min_cohort_date, @max_cohort_date = @result.rows.map { |r| r[0] }.minmax
+        @buckets = {}
+        @rows.each do |r|
+          @buckets[[r[0], r[1]]] = r[2]
+        end
 
-          num_cols = @cohort_dates.size
-          @columns = ["Cohort", "Users"] + num_cols.times.map { |i| "#{@conversion_period.titleize} #{i + 1}" }
-          rows = []
-          date_format = @cohort_period == "month" ? "%b %Y" : "%b %-e, %Y"
-          @cohort_dates.each do |date|
-            row = [date.strftime(date_format), @buckets[[date, 0]] || 0]
-
-            num_cols.times do |i|
-              if @today >= date + (@cohort_days * i)
-                row << (@buckets[[date, i + 1]] || 0)
-              end
+        @cohort_dates = []
+        current_date = @max_cohort_date
+        while current_date && current_date >= @min_cohort_date
+          @cohort_dates << current_date
+          current_date =
+            case @cohort_period
+            when "day"
+              current_date - 1
+            when "week"
+              current_date - 7
+            else
+              current_date.prev_month
             end
+        end
 
-            rows << row
+        num_cols = @cohort_dates.size
+        @columns = ["Cohort", "Users"] + num_cols.times.map { |i| "#{@conversion_period.titleize} #{i + 1}" }
+        rows = []
+        date_format = @cohort_period == "month" ? "%b %Y" : "%b %-e, %Y"
+        @cohort_dates.each do |date|
+          row = [date.strftime(date_format), @buckets[[date, 0]] || 0]
+
+          num_cols.times do |i|
+            if @today >= date + (@cohort_days * i)
+              row << (@buckets[[date, i + 1]] || 0)
+            end
           end
-          @rows = rows
+
+          rows << row
         end
+        @rows = rows
       end
+    end
   end
 end

data/app/controllers/blazer/uploads_controller.rb

@@ -86,62 +86,62 @@ module Blazer
 
     private
 
-      def update_file(upload, drop: nil)
-        file = params.require(:upload).fetch(:file)
-        raise Blazer::UploadError, "File is not a CSV" if file.content_type != "text/csv"
-        raise Blazer::UploadError, "File is too large (maximum is 100MB)" if file.size > 100.megabytes
-
-        contents = file.read
-        rows = CSV.parse(contents, converters: %i[numeric date date_time])
-
-        # friendly column names
-        columns = rows.shift.map { |v| v.to_s.encode("UTF-8").gsub("%", " pct ").parameterize.gsub("-", "_") }
-        duplicate_column = columns.find { |c| columns.count(c) > 1 }
-        raise Blazer::UploadError, "Duplicate column name: #{duplicate_column}" if duplicate_column
-
-        column_types =
-          columns.size.times.map do |i|
-            values = rows.map { |r| r[i] }.uniq.compact
-            if values.all? { |v| v.is_a?(Integer) && v >= -9223372036854775808 && v <= 9223372036854775807 }
-              "bigint"
-            elsif values.all? { |v| v.is_a?(Numeric) }
-              "decimal"
-            elsif values.all? { |v| v.is_a?(DateTime) }
-              "timestamptz"
-            elsif values.all? { |v| v.is_a?(Date) }
-              "date"
-            else
-              "text"
-            end
+    def update_file(upload, drop: nil)
+      file = params.require(:upload).fetch(:file)
+      raise Blazer::UploadError, "File is not a CSV" if file.content_type != "text/csv"
+      raise Blazer::UploadError, "File is too large (maximum is 100MB)" if file.size > 100.megabytes
+
+      contents = file.read
+      rows = CSV.parse(contents, converters: %i[numeric date date_time])
+
+      # friendly column names
+      columns = rows.shift.map { |v| v.to_s.encode("UTF-8").gsub("%", " pct ").parameterize.gsub("-", "_") }
+      duplicate_column = columns.find { |c| columns.count(c) > 1 }
+      raise Blazer::UploadError, "Duplicate column name: #{duplicate_column}" if duplicate_column
+
+      column_types =
+        columns.size.times.map do |i|
+          values = rows.map { |r| r[i] }.uniq.compact
+          if values.all? { |v| v.is_a?(Integer) && v >= -9223372036854775808 && v <= 9223372036854775807 }
+            "bigint"
+          elsif values.all? { |v| v.is_a?(Numeric) }
+            "decimal"
+          elsif values.all? { |v| v.is_a?(DateTime) }
+            "timestamptz"
+          elsif values.all? { |v| v.is_a?(Date) }
+            "date"
+          else
+            "text"
           end
+        end
 
-        begin
-          # maybe SET LOCAL statement_timeout = '30s'
-          # maybe regenerate CSV in Ruby to ensure consistent parsing
-          Blazer.uploads_connection.transaction do
-            Blazer.uploads_connection.execute("DROP TABLE IF EXISTS #{Blazer.uploads_table_name(drop)}") if drop
-            Blazer.uploads_connection.execute("CREATE TABLE #{upload.table_name} (#{columns.map.with_index { |c, i| "#{Blazer.uploads_connection.quote_column_name(c)} #{column_types[i]}" }.join(", ")})")
-            Blazer.uploads_connection.raw_connection.copy_data("COPY #{upload.table_name} FROM STDIN CSV HEADER") do
-              Blazer.uploads_connection.raw_connection.put_copy_data(contents)
-            end
+      begin
+        # maybe SET LOCAL statement_timeout = '30s'
+        # maybe regenerate CSV in Ruby to ensure consistent parsing
+        Blazer.uploads_connection.transaction do
+          Blazer.uploads_connection.execute("DROP TABLE IF EXISTS #{Blazer.uploads_table_name(drop)}") if drop
+          Blazer.uploads_connection.execute("CREATE TABLE #{upload.table_name} (#{columns.map.with_index { |c, i| "#{Blazer.uploads_connection.quote_column_name(c)} #{column_types[i]}" }.join(", ")})")
+          Blazer.uploads_connection.raw_connection.copy_data("COPY #{upload.table_name} FROM STDIN CSV HEADER") do
+            Blazer.uploads_connection.raw_connection.put_copy_data(contents)
           end
-        rescue ActiveRecord::StatementInvalid => e
-          raise Blazer::UploadError, "Table already exists" if e.message.include?("PG::DuplicateTable")
-          raise e
         end
+      rescue ActiveRecord::StatementInvalid => e
+        raise Blazer::UploadError, "Table already exists" if e.message.include?("PG::DuplicateTable")
+        raise e
       end
+    end
 
-      def upload_params
-        params.require(:upload).except(:file).permit(:table, :description)
-      end
+    def upload_params
+      params.require(:upload).except(:file).permit(:table, :description)
+    end
 
-      def set_upload
-        @upload = Blazer::Upload.find(params[:id])
-      end
+    def set_upload
+      @upload = Blazer::Upload.find(params[:id])
+    end
 
-      # routes aren't added, but also check here
-      def ensure_uploads
-        render plain: "Uploads not enabled" unless Blazer.uploads?
-      end
+    # routes aren't added, but also check here
+    def ensure_uploads
+      render plain: "Uploads not enabled" unless Blazer.uploads?
+    end
   end
 end

data/app/helpers/blazer/base_helper.rb

@@ -28,10 +28,6 @@ module Blazer
       end
     end
 
-    def blazer_maps?
-      Blazer.mapbox_access_token.present?
-    end
-
     def blazer_js_var(name, value)
       "var #{name} = #{json_escape(value.to_json(root: false))};".html_safe
     end