blazer 2.2.4 → 2.3.0

data/app/controllers/blazer/base_controller.rb

@@ -96,12 +96,12 @@ module Blazer
       # when permitted parameters are passed in Rails 6,
       # they appear to be added as GET parameters
       # root_url(params.permit(:host))
-      BLACKLISTED_KEYS = [:controller, :action, :id, :host, :query, :dashboard, :query_id, :query_ids, :table_names, :authenticity_token, :utf8, :_method, :commit, :statement, :data_source, :name, :fork_query_id, :blazer, :run_id, :script_name, :original_script_name]
+      UNPERMITTED_KEYS = [:controller, :action, :id, :host, :query, :dashboard, :query_id, :query_ids, :table_names, :authenticity_token, :utf8, :_method, :commit, :statement, :data_source, :name, :fork_query_id, :blazer, :run_id, :script_name, :original_script_name]
 
-      # remove blacklisted keys from both params and permitted keys for better sleep
+      # remove unpermitted keys from both params and permitted keys for better sleep
       def variable_params(resource)
-        permitted_keys = resource.variables - BLACKLISTED_KEYS.map(&:to_s)
-        params.except(*BLACKLISTED_KEYS).permit(*permitted_keys)
+        permitted_keys = resource.variables - UNPERMITTED_KEYS.map(&:to_s)
+        params.except(*UNPERMITTED_KEYS).slice(*permitted_keys).permit!
       end
       helper_method :variable_params
 

data/app/controllers/blazer/dashboards_controller.rb

@@ -21,8 +21,11 @@ module Blazer
 
     def show
       @queries = @dashboard.dashboard_queries.order(:position).preload(:query).map(&:query)
+      @statements = []
       @queries.each do |query|
-        process_vars(query.statement, query.data_source)
+        statement = query.statement.dup
+        process_vars(statement, query.data_source)
+        @statements << statement
       end
       @bind_vars ||= []
 
@@ -51,7 +54,7 @@ module Blazer
 
     def destroy
       @dashboard.destroy
-      redirect_to dashboards_path
+      redirect_to root_path
     end
 
     def refresh

data/app/controllers/blazer/queries_controller.rb

@@ -38,11 +38,18 @@ module Blazer
       if params[:fork_query_id]
         @query.statement ||= Blazer::Query.find(params[:fork_query_id]).try(:statement)
       end
+      if params[:upload_id]
+        upload = Blazer::Upload.find(params[:upload_id])
+        upload_settings = Blazer.settings["uploads"]
+        @query.data_source ||= upload_settings["data_source"]
+        @query.statement ||= "SELECT * FROM #{upload.table_name} LIMIT 10"
+      end
     end
 
     def create
       @query = Blazer::Query.new(query_params)
       @query.creator = blazer_user if @query.respond_to?(:creator)
+      @query.status = "active" if @query.respond_to?(:status)
 
       if @query.save
         redirect_to query_path(@query, variable_params(@query))
@@ -64,6 +71,8 @@ module Blazer
         @sql_errors << error if error
       end
 
+      @query.update!(status: "active") if @query.try(:status) == "archived"
+
       Blazer.transform_statement.call(data_source, @statement) if Blazer.transform_statement
     end
 
@@ -249,7 +258,9 @@ module Blazer
                 r[lat_index] && r[lon_index]
               end.map do |r|
                 {
-                  title: r.each_with_index.map{ |v, i| i == lat_index || i == lon_index ? nil : "<strong>#{@columns[i]}:</strong> #{v}" }.compact.join("<br />").truncate(140),
+                  # Mapbox.js does sanitization with https://github.com/mapbox/sanitize-caja
+                  # but we should do it here as well
+                  title: r.each_with_index.map { |v, i| i == lat_index || i == lon_index ? nil : "<strong>#{ERB::Util.html_escape(@columns[i])}:</strong> #{ERB::Util.html_escape(v)}" }.compact.join("<br />").truncate(140),
                   latitude: r[lat_index],
                   longitude: r[lon_index]
                 }
@@ -277,7 +288,7 @@ module Blazer
           @queries = queries_by_ids(Blazer::Audit.where(user_id: blazer_user.id).order(created_at: :desc).limit(500).pluck(:query_id).uniq)
         else
           @queries = @queries.limit(limit) if limit
-          @queries = @queries.order(:name)
+          @queries = @queries.active.order(:name)
         end
         @queries = @queries.to_a
 
@@ -298,7 +309,7 @@ module Blazer
       end
 
       def queries_by_ids(favorite_query_ids)
-        queries = Blazer::Query.named.where(id: favorite_query_ids)
+        queries = Blazer::Query.active.named.where(id: favorite_query_ids)
         queries = queries.includes(:creator) if Blazer.user_class
         queries = queries.index_by(&:id)
         favorite_query_ids.map { |query_id| queries[query_id] }.compact

data/app/controllers/blazer/uploads_controller.rb

@@ -0,0 +1,136 @@
+module Blazer
+  class UploadsController < BaseController
+    before_action :ensure_uploads
+    before_action :set_upload, only: [:show, :edit, :update, :destroy]
+
+    def index
+      @uploads = Blazer::Upload.order(:table)
+    end
+
+    def new
+      @upload = Blazer::Upload.new
+    end
+
+    def create
+      @upload = Blazer::Upload.new(upload_params)
+      # use creator_id instead of creator
+      # since we setup association without checking if column exists
+      @upload.creator = blazer_user if @upload.respond_to?(:creator_id=) && blazer_user
+
+      success = params.require(:upload).key?(:file)
+      if success
+        Blazer::Upload.transaction do
+          success = @upload.save
+          if success
+            begin
+              update_file(@upload)
+            rescue CSV::MalformedCSVError, Blazer::UploadError => e
+              @upload.errors.add(:base, e.message)
+              success = false
+              raise ActiveRecord::Rollback
+            end
+          end
+        end
+      else
+        @upload.errors.add(:base, "File can't be blank")
+      end
+
+      if success
+        redirect_to upload_path(@upload)
+      else
+        render_errors @upload
+      end
+    end
+
+    def show
+      redirect_to new_query_path(upload_id: @upload.id)
+    end
+
+    def edit
+    end
+
+    def update
+      original_table = @upload.table
+      @upload.assign_attributes(upload_params)
+
+      success = nil
+      Blazer::Upload.transaction do
+        success = @upload.save
+        if success
+          if params.require(:upload).key?(:file)
+            begin
+              update_file(@upload, drop: original_table)
+            rescue CSV::MalformedCSVError, Blazer::UploadError => e
+              @upload.errors.add(:base, e.message)
+              success = false
+              raise ActiveRecord::Rollback
+            end
+          elsif @upload.table != original_table
+            Blazer.uploads_connection.execute("ALTER TABLE #{Blazer.uploads_table_name(original_table)} RENAME TO #{Blazer.uploads_connection.quote_table_name(@upload.table)}")
+          end
+        end
+      end
+
+      if success
+        redirect_to upload_path(@upload)
+      else
+        render_errors @upload
+      end
+    end
+
+    def destroy
+      Blazer.uploads_connection.execute("DROP TABLE IF EXISTS #{@upload.table_name}")
+      @upload.destroy
+      redirect_to uploads_path
+    end
+
+    private
+
+      def update_file(upload, drop: nil)
+        file = params.require(:upload).fetch(:file)
+        raise Blazer::UploadError, "File is not a CSV" if file.content_type != "text/csv"
+        raise Blazer::UploadError, "File is too large (maximum is 100MB)" if file.size > 100.megabytes
+
+        contents = file.read
+        rows = CSV.parse(contents, converters: %i[numeric date date_time])
+        columns = rows.shift.map(&:to_s)
+        column_types =
+          columns.size.times.map do |i|
+            values = rows.map { |r| r[i] }.uniq.compact
+            if values.all? { |v| v.is_a?(Integer) && v >= -9223372036854775808 && v <= 9223372036854775807 }
+              "bigint"
+            elsif values.all? { |v| v.is_a?(Numeric) }
+              "decimal"
+            elsif values.all? { |v| v.is_a?(DateTime) }
+              "timestamptz"
+            elsif values.all? { |v| v.is_a?(Date) }
+              "date"
+            else
+              "text"
+            end
+          end
+
+        # maybe SET LOCAL statement_timeout = '30s'
+        Blazer.uploads_connection.transaction do
+          Blazer.uploads_connection.execute("DROP TABLE IF EXISTS #{Blazer.uploads_table_name(drop)}") if drop
+          Blazer.uploads_connection.execute("CREATE TABLE #{upload.table_name} (#{columns.map.with_index { |c, i| "#{Blazer.uploads_connection.quote_column_name(c)} #{column_types[i]}" }.join(", ")})")
+          Blazer.uploads_connection.raw_connection.copy_data("COPY #{upload.table_name} FROM STDIN CSV HEADER") do
+            Blazer.uploads_connection.raw_connection.put_copy_data(contents)
+          end
+        end
+      end
+
+      def upload_params
+        params.require(:upload).except(:file).permit(:table, :description)
+      end
+
+      def set_upload
+        @upload = Blazer::Upload.find(params[:id])
+      end
+
+      # routes aren't added, but also check here
+      def ensure_uploads
+        render plain: "Uploads not enabled" unless Blazer.uploads?
+      end
+  end
+end

data/app/mailers/blazer/slack_notifier.rb

@@ -62,7 +62,7 @@ module Blazer
 
     # checks shouldn't have variables, but in any case,
     # avoid passing variable params to url helpers
-    # (known unsafe parameters are removed, but blacklist isn't ideal)
+    # (known unsafe parameters are removed, but still not ideal)
     def self.query_url(id)
       Blazer::Engine.routes.url_helpers.query_url(id, ActionMailer::Base.default_url_options)
     end

data/app/models/blazer/query.rb

@@ -8,6 +8,7 @@ module Blazer
 
     validates :statement, presence: true
 
+    scope :active, -> { column_names.include?("status") ? where(status: "active") : all }
     scope :named, -> { where("blazer_queries.name <> ''") }
 
     def to_param

data/app/models/blazer/upload.rb

@@ -0,0 +1,11 @@
+module Blazer
+  class Upload < Record
+    belongs_to :creator, optional: true, class_name: Blazer.user_class.to_s if Blazer.user_class
+
+    validates :table, presence: true, uniqueness: true, format: {with: /\A[a-z0-9_]+\z/, message: "can only contain lowercase letters, numbers, and underscores"}, length: {maximum: 63}
+
+    def table_name
+      Blazer.uploads_table_name(table)
+    end
+  end
+end

data/app/models/blazer/uploads_connection.rb

@@ -0,0 +1,7 @@
+module Blazer
+  class UploadsConnection < ActiveRecord::Base
+    self.abstract_class = true
+
+    establish_connection Blazer.settings["uploads"]["url"] if Blazer.uploads?
+  end
+end

data/app/views/blazer/_nav.html.erb

@@ -6,6 +6,9 @@
   </button>
   <ul class="dropdown-menu">
     <li><%= link_to "Checks", checks_path %></li>
+    <% if Blazer.uploads? %>
+      <li><%= link_to "Uploads", uploads_path %></li>
+    <% end %>
     <li role="separator" class="divider"></li>
     <li><%= link_to "New Query", new_query_path %></li>
     <li><%= link_to "New Dashboard", new_dashboard_path %></li>

data/app/views/blazer/_variables.html.erb

@@ -42,6 +42,7 @@
               singleDatePicker: true,
               locale: {format: format},
               autoUpdateInput: false,
+              autoApply: true,
               startDate: input.val().length > 0 ? moment.tz(input.val(), timeZone) : now
             })
             // hack to start with empty date
@@ -95,7 +96,8 @@
             },
             startDate: dateStr(29),
             endDate: dateStr(),
-            opens: "right"
+            opens: "right",
+            alwaysShowCalendars: true
           },
           function(start, end) {
             setTimeInputs(start, end)

data/app/views/blazer/checks/_form.html.erb

@@ -13,7 +13,7 @@
       <%= f.select :query_id, [], {include_blank: true} %>
     </div>
     <script>
-      <%= blazer_js_var "queries", Blazer::Query.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} } %>
+      <%= blazer_js_var "queries", Blazer::Query.active.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} } %>
       <%= blazer_js_var "items", [@check.query_id].compact %>
 
       $("#check_query_id").selectize({options: queries, items: items, highlight: false, maxOptions: 100}).parents(".hide").removeClass("hide");

data/app/views/blazer/checks/index.html.erb

@@ -10,6 +10,9 @@
       </button>
       <ul class="dropdown-menu">
         <li><%= link_to "Home", root_path %></li>
+        <% if Blazer.uploads? %>
+          <li><%= link_to "Uploads", uploads_path %></li>
+        <% end %>
         <li role="separator" class="divider"></li>
         <li><%= link_to "New Query", new_query_path %></li>
         <li><%= link_to "New Dashboard", new_dashboard_path %></li>

data/app/views/blazer/dashboards/_form.html.erb

@@ -31,7 +31,7 @@
 <% end %>
 
 <script>
-  <%= blazer_js_var "queries", Blazer::Query.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} } %>
+  <%= blazer_js_var "queries", Blazer::Query.active.named.order(:name).select("id, name").map { |q| {text: q.name, value: q.id} } %>
   <%= blazer_js_var "dashboardQueries", @queries || @dashboard.dashboard_queries.order(:position).map(&:query) %>
 
   var app = new Vue({

data/app/views/blazer/dashboards/show.html.erb

@@ -39,7 +39,7 @@
     </div>
   </div>
   <script>
-    <%= blazer_js_var "data", {statement: query.statement, query_id: query.id, data_source: query.data_source, only_chart: true} %>
+    <%= blazer_js_var "data", {statement: @statements[i], query_id: query.id, data_source: query.data_source, only_chart: true} %>
 
     runQuery(data, function (data) {
       $("#chart-<%= i %>").html(data)

data/app/views/blazer/queries/home.html.erb

@@ -19,6 +19,9 @@
         </button>
         <ul class="dropdown-menu">
           <li><%= link_to "Checks", checks_path %></li>
+          <% if Blazer.uploads? %>
+            <li><%= link_to "Uploads", uploads_path %></li>
+          <% end %>
           <li role="separator" class="divider"></li>
           <li><%= link_to "New Dashboard", new_dashboard_path %></li>
           <li><%= link_to "New Check", new_check_path %></li>

data/app/views/blazer/queries/run.html.erb

@@ -80,7 +80,8 @@
         <%= blazer_js_var "mapboxAccessToken", Blazer.mapbox_access_token %>
         <%= blazer_js_var "markers", @markers %>
         L.mapbox.accessToken = mapboxAccessToken;
-        var map = L.mapbox.map('map', 'mapbox.streets');
+        var map = L.mapbox.map('map')
+          .addLayer(L.mapbox.styleLayer('mapbox://styles/mapbox/streets-v11'));
         var featureLayer = L.mapbox.featureLayer().addTo(map);
         var geojson = [];
         for (var i = 0; i < markers.length; i++) {
@@ -106,13 +107,13 @@
       </script>
     <% elsif chart_type == "line" %>
       <% chart_data = @columns[1..-1].each_with_index.map{ |k, i| {name: blazer_series_name(k), data: @rows.map{ |r| [r[0], r[i + 1]] }, library: series_library[i]} } %>
-      <%= line_chart chart_data, chart_options %>
+      <%= line_chart chart_data, **chart_options %>
     <% elsif chart_type == "line2" %>
-      <%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, chart_options %>
+      <%= line_chart @rows.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.map { |v2| [v2[0], v2[2]] }, library: series_library[i]} }, **chart_options %>
     <% elsif chart_type == "pie" %>
-      <%= pie_chart @rows.map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[1]] }, chart_options %>
+      <%= pie_chart @rows.map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[1]] }, **chart_options %>
     <% elsif chart_type == "bar" %>
-      <%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: blazer_series_name(name), data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, chart_options %>
+      <%= column_chart (values.size - 1).times.map { |i| name = @columns[i + 1]; {name: blazer_series_name(name), data: @rows.first(20).map { |r| [(@boom[@columns[0]] || {})[r[0].to_s] || r[0], r[i + 1]] } } }, **chart_options %>
     <% elsif chart_type == "bar2" %>
       <% first_20 = @rows.group_by { |r| r[0] }.values.first(20).flatten(1) %>
       <% labels = first_20.map { |r| r[0] }.uniq %>
@@ -122,7 +123,7 @@
           <% first_20 << [l, s, 0] unless first_20.find { |r| r[0] == l && r[1] == s } %>
         <% end %>
       <% end %>
-      <%= column_chart first_20.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.sort_by { |r2| labels.index(r2[0]) }.map { |v2| v3 = v2[0]; [(@boom[@columns[0]] || {})[v3.to_s] || v3, v2[2]] }} }, chart_options %>
+      <%= column_chart first_20.group_by { |r| v = r[1]; (@boom[@columns[1]] || {})[v.to_s] || v }.each_with_index.map { |(name, v), i| {name: blazer_series_name(name), data: v.sort_by { |r2| labels.index(r2[0]) }.map { |v2| v3 = v2[0]; [(@boom[@columns[0]] || {})[v3.to_s] || v3, v2[2]] }} }, **chart_options %>
     <% elsif chart_type == "scatter" %>
       <%= scatter_chart @rows, xtitle: @columns[0], ytitle: @columns[1], **chart_options %>
     <% elsif @only_chart %>

data/app/views/blazer/queries/show.html.erb

@@ -56,7 +56,9 @@
       $("#results").addClass("query-error").html(message)
     }
 
-    <%= blazer_js_var "data", variable_params(@query).merge(statement: @statement, query_id: @query.id, data_source: @query.data_source) %>
+    <% data = variable_params(@query).merge(statement: @statement, query_id: @query.id, data_source: @query.data_source) %>
+    <% data.merge!(forecast: "t") if params[:forecast] %>
+    <%= blazer_js_var "data", data %>
 
     runQuery(data, showRun, showError)
   </script>

data/app/views/blazer/uploads/_form.html.erb

@@ -0,0 +1,27 @@
+<%= form_for @upload, html: {class: "small-form"} do |f| %>
+  <% if @upload.errors.any? %>
+    <div class="alert alert-danger"><%= @upload.errors.full_messages.first %></div>
+  <% elsif !@upload.persisted? %>
+    <p>Create a database table from a CSV file. The table will be created in the <code><%= Blazer.settings["uploads"]["schema"] %></code> schema.</p>
+  <% end %>
+
+  <div class="form-group">
+    <%= f.label :table %>
+    <%= f.text_field :table, class: "form-control" %>
+  </div>
+  <div class="form-group">
+    <%= f.label :description %>
+    <%= f.text_area :description, placeholder: "Optional", style: "height: 60px;", class: "form-control" %>
+  </div>
+  <div class="form-group">
+    <%= f.label :file %>
+    <%= f.file_field :file, accept: "text/csv", style: "margin-top: 6px; margin-bottom: 21px;" %>
+  </div>
+  <p>
+    <% if @upload.persisted? %>
+      <%= link_to "Delete", upload_path(@upload), method: :delete, "data-confirm" => "Are you sure?", class: "btn btn-danger" %>
+    <% end %>
+    <%= f.submit "Save", class: "btn btn-success" %>
+    <%= link_to "Back", :back, class: "btn btn-link" %>
+  </p>
+<% end %>

data/app/views/blazer/uploads/edit.html.erb

@@ -0,0 +1,3 @@
+<% blazer_title "Edit Upload" %>
+
+<%= render partial: "form" %>

data/app/views/blazer/uploads/index.html.erb

@@ -0,0 +1,55 @@
+<% blazer_title "Uploads" %>
+
+<div id="header">
+  <div class="pull-right" style="line-height: 34px;">
+    <div class="btn-group">
+      <%= link_to "New Upload", new_upload_path, class: "btn btn-info" %>
+      <button type="button" class="btn btn-info dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+        <span class="caret"></span>
+        <span class="sr-only">Toggle Dropdown</span>
+      </button>
+      <ul class="dropdown-menu">
+        <li><%= link_to "Home", root_path %></li>
+        <li><%= link_to "Checks", checks_path %></li>
+        <li role="separator" class="divider"></li>
+        <li><%= link_to "New Query", new_query_path %></li>
+        <li><%= link_to "New Dashboard", new_dashboard_path %></li>
+        <li><%= link_to "New Check", new_check_path %></li>
+      </ul>
+    </div>
+  </div>
+
+  <input id="search" type="text" placeholder="Start typing a table or person" style="width: 300px; display: inline-block;" class="search form-control" />
+</div>
+
+<table id="uploads" class="table">
+  <thead>
+    <tr>
+      <th>Table</th>
+      <th style="width: 60%;"></th>
+      <% if Blazer.user_class %>
+        <th style="width: 20%; text-align: right;">Mastermind</th>
+      <% end%>
+    </tr>
+  </thead>
+  <tbody>
+    <% @uploads.each do |upload| %>
+      <tr>
+        <td><%= link_to upload.table, edit_upload_path(upload) %></td>
+        <td><%= truncate(upload.description, length: 100, separator: " ") %></td>
+        <% if Blazer.user_class %>
+          <td class="creator"><%= blazer_user && upload.creator == blazer_user ? "You" : upload.creator.try(Blazer.user_name) %></td>
+        <% end %>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+
+<script>
+  $("#search").on("keyup", function() {
+    var value = $(this).val().toLowerCase()
+    $("#uploads tbody tr").filter( function() {
+      $(this).toggle($(this).text().toLowerCase().indexOf(value) > -1)
+    })
+  }).focus()
+</script>