blazer 1.7.7 → 2.6.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (139) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +242 -33
  3. data/CONTRIBUTING.md +42 -0
  4. data/LICENSE.txt +1 -1
  5. data/README.md +621 -211
  6. data/app/assets/fonts/blazer/glyphicons-halflings-regular.eot +0 -0
  7. data/app/assets/fonts/blazer/glyphicons-halflings-regular.svg +0 -0
  8. data/app/assets/fonts/blazer/glyphicons-halflings-regular.ttf +0 -0
  9. data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff +0 -0
  10. data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff2 +0 -0
  11. data/app/assets/images/blazer/favicon.png +0 -0
  12. data/app/assets/javascripts/blazer/Chart.js +15658 -10011
  13. data/app/assets/javascripts/blazer/Sortable.js +3413 -848
  14. data/app/assets/javascripts/blazer/ace/ace.js +21294 -4
  15. data/app/assets/javascripts/blazer/ace/ext-language_tools.js +1991 -3
  16. data/app/assets/javascripts/blazer/ace/mode-sql.js +110 -1
  17. data/app/assets/javascripts/blazer/ace/snippets/sql.js +40 -1
  18. data/app/assets/javascripts/blazer/ace/snippets/text.js +14 -1
  19. data/app/assets/javascripts/blazer/ace/theme-twilight.js +116 -1
  20. data/app/assets/javascripts/blazer/application.js +5 -3
  21. data/app/assets/javascripts/blazer/bootstrap.js +842 -628
  22. data/app/assets/javascripts/blazer/chartkick.js +2015 -1244
  23. data/app/assets/javascripts/blazer/daterangepicker.js +372 -299
  24. data/app/assets/javascripts/blazer/highlight.min.js +3 -0
  25. data/app/assets/javascripts/blazer/{jquery_ujs.js → jquery-ujs.js} +161 -75
  26. data/app/assets/javascripts/blazer/jquery.js +10126 -9562
  27. data/app/assets/javascripts/blazer/jquery.stickytableheaders.js +321 -259
  28. data/app/assets/javascripts/blazer/moment-timezone-with-data.js +1546 -0
  29. data/app/assets/javascripts/blazer/moment.js +5085 -2460
  30. data/app/assets/javascripts/blazer/queries.js +18 -4
  31. data/app/assets/javascripts/blazer/routes.js +3 -0
  32. data/app/assets/javascripts/blazer/selectize.js +3828 -3604
  33. data/app/assets/javascripts/blazer/stupidtable-custom-settings.js +13 -0
  34. data/app/assets/javascripts/blazer/stupidtable.js +254 -87
  35. data/app/assets/javascripts/blazer/vue.js +11175 -6676
  36. data/app/assets/stylesheets/blazer/application.css +51 -6
  37. data/app/assets/stylesheets/blazer/bootstrap-propshaft.css +10 -0
  38. data/app/assets/stylesheets/blazer/bootstrap-sprockets.css.erb +10 -0
  39. data/app/assets/stylesheets/blazer/{bootstrap.css.erb → bootstrap.css} +1337 -711
  40. data/app/assets/stylesheets/blazer/{daterangepicker-bs3.css → daterangepicker.css} +207 -172
  41. data/app/assets/stylesheets/blazer/{selectize.default.css → selectize.css} +26 -10
  42. data/app/controllers/blazer/base_controller.rb +73 -46
  43. data/app/controllers/blazer/checks_controller.rb +1 -1
  44. data/app/controllers/blazer/dashboards_controller.rb +7 -13
  45. data/app/controllers/blazer/queries_controller.rb +171 -51
  46. data/app/controllers/blazer/uploads_controller.rb +147 -0
  47. data/app/helpers/blazer/base_helper.rb +6 -16
  48. data/app/models/blazer/audit.rb +3 -3
  49. data/app/models/blazer/check.rb +31 -5
  50. data/app/models/blazer/dashboard.rb +6 -2
  51. data/app/models/blazer/dashboard_query.rb +1 -1
  52. data/app/models/blazer/query.rb +30 -4
  53. data/app/models/blazer/record.rb +5 -0
  54. data/app/models/blazer/upload.rb +11 -0
  55. data/app/models/blazer/uploads_connection.rb +7 -0
  56. data/app/views/blazer/_nav.html.erb +3 -1
  57. data/app/views/blazer/_variables.html.erb +48 -23
  58. data/app/views/blazer/check_mailer/failing_checks.html.erb +1 -0
  59. data/app/views/blazer/check_mailer/state_change.html.erb +1 -0
  60. data/app/views/blazer/checks/_form.html.erb +17 -9
  61. data/app/views/blazer/checks/edit.html.erb +2 -0
  62. data/app/views/blazer/checks/index.html.erb +37 -5
  63. data/app/views/blazer/checks/new.html.erb +2 -0
  64. data/app/views/blazer/dashboards/_form.html.erb +5 -5
  65. data/app/views/blazer/dashboards/edit.html.erb +2 -0
  66. data/app/views/blazer/dashboards/new.html.erb +2 -0
  67. data/app/views/blazer/dashboards/show.html.erb +13 -7
  68. data/app/views/blazer/queries/_caching.html.erb +16 -0
  69. data/app/views/blazer/queries/_cohorts.html.erb +48 -0
  70. data/app/views/blazer/queries/_form.html.erb +23 -13
  71. data/app/views/blazer/queries/docs.html.erb +137 -0
  72. data/app/views/blazer/queries/home.html.erb +21 -7
  73. data/app/views/blazer/queries/run.html.erb +64 -29
  74. data/app/views/blazer/queries/schema.html.erb +44 -7
  75. data/app/views/blazer/queries/show.html.erb +15 -8
  76. data/app/views/blazer/uploads/_form.html.erb +27 -0
  77. data/app/views/blazer/uploads/edit.html.erb +3 -0
  78. data/app/views/blazer/uploads/index.html.erb +55 -0
  79. data/app/views/blazer/uploads/new.html.erb +3 -0
  80. data/app/views/layouts/blazer/application.html.erb +10 -5
  81. data/config/routes.rb +10 -1
  82. data/lib/blazer/adapters/athena_adapter.rb +182 -0
  83. data/lib/blazer/adapters/base_adapter.rb +24 -1
  84. data/lib/blazer/adapters/bigquery_adapter.rb +79 -0
  85. data/lib/blazer/adapters/cassandra_adapter.rb +70 -0
  86. data/lib/blazer/adapters/drill_adapter.rb +38 -0
  87. data/lib/blazer/adapters/druid_adapter.rb +102 -0
  88. data/lib/blazer/adapters/elasticsearch_adapter.rb +30 -18
  89. data/lib/blazer/adapters/hive_adapter.rb +55 -0
  90. data/lib/blazer/adapters/ignite_adapter.rb +64 -0
  91. data/lib/blazer/adapters/influxdb_adapter.rb +57 -0
  92. data/lib/blazer/adapters/mongodb_adapter.rb +5 -1
  93. data/lib/blazer/adapters/neo4j_adapter.rb +62 -0
  94. data/lib/blazer/adapters/opensearch_adapter.rb +52 -0
  95. data/lib/blazer/adapters/presto_adapter.rb +9 -0
  96. data/lib/blazer/adapters/salesforce_adapter.rb +50 -0
  97. data/lib/blazer/adapters/snowflake_adapter.rb +82 -0
  98. data/lib/blazer/adapters/soda_adapter.rb +105 -0
  99. data/lib/blazer/adapters/spark_adapter.rb +14 -0
  100. data/lib/blazer/adapters/sql_adapter.rb +187 -20
  101. data/{app/mailers → lib}/blazer/check_mailer.rb +0 -0
  102. data/lib/blazer/data_source.rb +107 -30
  103. data/lib/blazer/engine.rb +21 -23
  104. data/lib/blazer/result.rb +95 -29
  105. data/lib/blazer/run_statement.rb +8 -4
  106. data/lib/blazer/run_statement_job.rb +8 -9
  107. data/lib/blazer/slack_notifier.rb +94 -0
  108. data/lib/blazer/statement.rb +75 -0
  109. data/lib/blazer/version.rb +1 -1
  110. data/lib/blazer.rb +154 -26
  111. data/lib/generators/blazer/install_generator.rb +7 -18
  112. data/lib/generators/blazer/templates/{config.yml → config.yml.tt} +26 -3
  113. data/lib/generators/blazer/templates/{install.rb → install.rb.tt} +6 -4
  114. data/lib/generators/blazer/templates/uploads.rb.tt +10 -0
  115. data/lib/generators/blazer/uploads_generator.rb +18 -0
  116. data/lib/tasks/blazer.rake +11 -1
  117. data/licenses/LICENSE-ace.txt +24 -0
  118. data/licenses/LICENSE-bootstrap.txt +21 -0
  119. data/licenses/LICENSE-chart.js.txt +9 -0
  120. data/licenses/LICENSE-chartkick.js.txt +22 -0
  121. data/licenses/LICENSE-daterangepicker.txt +21 -0
  122. data/licenses/LICENSE-fuzzysearch.txt +20 -0
  123. data/licenses/LICENSE-highlight.js.txt +29 -0
  124. data/licenses/LICENSE-jquery-ujs.txt +20 -0
  125. data/licenses/LICENSE-jquery.txt +20 -0
  126. data/licenses/LICENSE-moment-timezone.txt +20 -0
  127. data/licenses/LICENSE-moment.txt +22 -0
  128. data/licenses/LICENSE-selectize.txt +202 -0
  129. data/licenses/LICENSE-sortable.txt +21 -0
  130. data/licenses/LICENSE-stickytableheaders.txt +20 -0
  131. data/licenses/LICENSE-stupidtable.txt +19 -0
  132. data/licenses/LICENSE-vue.txt +21 -0
  133. metadata +83 -53
  134. data/.gitignore +0 -14
  135. data/Gemfile +0 -4
  136. data/Rakefile +0 -1
  137. data/app/assets/javascripts/blazer/highlight.pack.js +0 -1
  138. data/app/assets/javascripts/blazer/moment-timezone.js +0 -1007
  139. data/blazer.gemspec +0 -26
@@ -0,0 +1,105 @@
1
+ module Blazer
2
+ module Adapters
3
+ class SodaAdapter < BaseAdapter
4
+ def run_statement(statement, comment)
5
+ require "json"
6
+ require "net/http"
7
+ require "uri"
8
+
9
+ columns = []
10
+ rows = []
11
+ error = nil
12
+
13
+ # remove comments manually
14
+ statement = statement.gsub(/--.+/, "")
15
+ # only supports single line /* */ comments
16
+ # regex not perfect, but should be good enough
17
+ statement = statement.gsub(/\/\*.+\*\//, "")
18
+
19
+ # remove trailing semicolon
20
+ statement = statement.sub(/;\s*\z/, "")
21
+
22
+ # remove whitespace
23
+ statement = statement.squish
24
+
25
+ uri = URI(settings["url"])
26
+ uri.query = URI.encode_www_form("$query" => statement)
27
+
28
+ req = Net::HTTP::Get.new(uri)
29
+ req["X-App-Token"] = settings["app_token"] if settings["app_token"]
30
+
31
+ options = {
32
+ use_ssl: uri.scheme == "https",
33
+ open_timeout: 3,
34
+ read_timeout: 30
35
+ }
36
+
37
+ begin
38
+ # use Net::HTTP instead of soda-ruby for types and better error messages
39
+ res = Net::HTTP.start(uri.hostname, uri.port, options) do |http|
40
+ http.request(req)
41
+ end
42
+
43
+ if res.is_a?(Net::HTTPSuccess)
44
+ body = JSON.parse(res.body)
45
+
46
+ columns = JSON.parse(res["x-soda2-fields"])
47
+ column_types = columns.zip(JSON.parse(res["x-soda2-types"])).to_h
48
+
49
+ columns.reject! { |f| f.start_with?(":@") }
50
+ # rows can be missing some keys in JSON, so need to map by column
51
+ rows = body.map { |r| columns.map { |c| r[c] } }
52
+
53
+ columns.each_with_index do |column, i|
54
+ # nothing to do for boolean
55
+ case column_types[column]
56
+ when "number"
57
+ # check if likely an integer column
58
+ if rows.all? { |r| r[i].to_i == r[i].to_f }
59
+ rows.each do |row|
60
+ row[i] = row[i].to_i
61
+ end
62
+ else
63
+ rows.each do |row|
64
+ row[i] = row[i].to_f
65
+ end
66
+ end
67
+ when "floating_timestamp"
68
+ # check if likely a date column
69
+ if rows.all? { |r| r[i].end_with?("T00:00:00.000") }
70
+ rows.each do |row|
71
+ row[i] = Date.parse(row[i])
72
+ end
73
+ else
74
+ utc = ActiveSupport::TimeZone["Etc/UTC"]
75
+ rows.each do |row|
76
+ row[i] = utc.parse(row[i])
77
+ end
78
+ end
79
+ end
80
+ end
81
+ else
82
+ error = JSON.parse(res.body)["message"] rescue "Bad response: #{res.code}"
83
+ end
84
+ rescue => e
85
+ error = e.message
86
+ end
87
+
88
+ [columns, rows, error]
89
+ end
90
+
91
+ def preview_statement
92
+ "SELECT * LIMIT 10"
93
+ end
94
+
95
+ def tables
96
+ ["all"]
97
+ end
98
+
99
+ # https://dev.socrata.com/docs/datatypes/text.html
100
+ def quoting
101
+ :single_quote_escape
102
+ end
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,14 @@
1
+ module Blazer
2
+ module Adapters
3
+ class SparkAdapter < HiveAdapter
4
+ def tables
5
+ client.execute("SHOW TABLES").map { |r| r["tableName"] }
6
+ end
7
+
8
+ # https://spark.apache.org/docs/latest/sql-ref-literals.html
9
+ def quoting
10
+ :backslash_escape
11
+ end
12
+ end
13
+ end
14
+ end
@@ -15,7 +15,7 @@ module Blazer
15
15
  end
16
16
  end
17
17
 
18
- def run_statement(statement, comment)
18
+ def run_statement(statement, comment, bind_params = [])
19
19
  columns = []
20
20
  rows = []
21
21
  error = nil
@@ -24,16 +24,17 @@ module Blazer
24
24
  in_transaction do
25
25
  set_timeout(data_source.timeout) if data_source.timeout
26
26
 
27
- result = select_all("#{statement} /*#{comment}*/")
27
+ binds = bind_params.map { |v| ActiveRecord::Relation::QueryAttribute.new(nil, v, ActiveRecord::Type::Value.new) }
28
+ result = connection_model.connection.select_all("#{statement} /*#{comment}*/", nil, binds)
28
29
  columns = result.columns
29
- cast_method = Rails::VERSION::MAJOR < 5 ? :type_cast : :cast_value
30
30
  result.rows.each do |untyped_row|
31
- rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] ? result.column_types[c].send(cast_method, untyped_row[i]) : untyped_row[i] })
31
+ rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] && result.column_types[c] ? result.column_types[c].send(:cast_value, untyped_row[i]) : untyped_row[i] })
32
32
  end
33
33
  end
34
34
  rescue => e
35
35
  error = e.message.sub(/.+ERROR: /, "")
36
36
  error = Blazer::TIMEOUT_MESSAGE if Blazer::TIMEOUT_ERRORS.any? { |e| error.include?(e) }
37
+ error = Blazer::VARIABLE_MESSAGE if error.include?("syntax error at or near \"$") || error.include?("Incorrect syntax near '@") || error.include?("your MySQL server version for the right syntax to use near '?")
37
38
  reconnect if error.include?("PG::ConnectionBad")
38
39
  end
39
40
 
@@ -41,18 +42,38 @@ module Blazer
41
42
  end
42
43
 
43
44
  def tables
44
- result = data_source.run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_name FROM information_schema.tables WHERE table_schema IN (?) ORDER BY table_name", schemas]), refresh_cache: true)
45
- result.rows.map(&:first)
45
+ sql = add_schemas("SELECT table_schema, table_name FROM information_schema.tables")
46
+ result = data_source.run_statement(sql, refresh_cache: true)
47
+ if postgresql? || redshift? || snowflake?
48
+ result.rows.sort_by { |r| [r[0] == default_schema ? "" : r[0], r[1]] }.map do |row|
49
+ table =
50
+ if row[0] == default_schema
51
+ row[1]
52
+ else
53
+ "#{row[0]}.#{row[1]}"
54
+ end
55
+
56
+ table = table.downcase if snowflake?
57
+
58
+ {
59
+ table: table,
60
+ value: connection_model.connection.quote_table_name(table)
61
+ }
62
+ end
63
+ else
64
+ result.rows.map(&:second).sort
65
+ end
46
66
  end
47
67
 
48
68
  def schema
49
- result = data_source.run_statement(connection_model.send(:sanitize_sql_array, ["SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns WHERE table_schema IN (?) ORDER BY 1, 2", schemas]))
50
- result.rows.group_by { |r| [r[0], r[1]] }.map { |k, vs| {schema: k[0], table: k[1], columns: vs.sort_by { |v| v[2] }.map { |v| {name: v[2], data_type: v[3]} }} }
69
+ sql = add_schemas("SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns")
70
+ result = data_source.run_statement(sql)
71
+ result.rows.group_by { |r| [r[0], r[1]] }.map { |k, vs| {schema: k[0], table: k[1], columns: vs.sort_by { |v| v[2] }.map { |v| {name: v[2], data_type: v[3]} }} }.sort_by { |t| [t[:schema] == default_schema ? "" : t[:schema], t[:table]] }
51
72
  end
52
73
 
53
74
  def preview_statement
54
- if postgresql?
55
- "SELECT * FROM \"{table}\" LIMIT 10"
75
+ if sqlserver?
76
+ "SELECT TOP (10) * FROM {table}"
56
77
  else
57
78
  "SELECT * FROM {table} LIMIT 10"
58
79
  end
@@ -64,13 +85,25 @@ module Blazer
64
85
 
65
86
  def cost(statement)
66
87
  result = explain(statement)
67
- match = /cost=\d+\.\d+..(\d+\.\d+) /.match(result)
68
- match[1] if match
88
+ if sqlserver?
89
+ result["TotalSubtreeCost"]
90
+ else
91
+ match = /cost=\d+\.\d+..(\d+\.\d+) /.match(result)
92
+ match[1] if match
93
+ end
69
94
  end
70
95
 
71
96
  def explain(statement)
72
97
  if postgresql? || redshift?
73
98
  select_all("EXPLAIN #{statement}").rows.first.first
99
+ elsif sqlserver?
100
+ begin
101
+ execute("SET SHOWPLAN_ALL ON")
102
+ result = select_all(statement).each.first
103
+ ensure
104
+ execute("SET SHOWPLAN_ALL OFF")
105
+ end
106
+ result
74
107
  end
75
108
  rescue
76
109
  nil
@@ -78,9 +111,9 @@ module Blazer
78
111
 
79
112
  def cancel(run_id)
80
113
  if postgresql?
81
- select_all("SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND query LIKE '%,run_id:#{run_id}%'")
114
+ select_all("SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND query LIKE ?", ["%,run_id:#{run_id}%"])
82
115
  elsif redshift?
83
- first_row = select_all("SELECT pid FROM stv_recents WHERE status = 'Running' AND query LIKE '%,run_id:#{run_id}%'").first
116
+ first_row = select_all("SELECT pid FROM stv_recents WHERE status = 'Running' AND query LIKE ?", ["%,run_id:#{run_id}%"]).first
84
117
  if first_row
85
118
  select_all("CANCEL #{first_row["pid"].to_i}")
86
119
  end
@@ -91,9 +124,95 @@ module Blazer
91
124
  !%w[CREATE ALTER UPDATE INSERT DELETE].include?(statement.split.first.to_s.upcase)
92
125
  end
93
126
 
127
+ def supports_cohort_analysis?
128
+ postgresql? || mysql?
129
+ end
130
+
131
+ # TODO treat date columns as already in time zone
132
+ def cohort_analysis_statement(statement, period:, days:)
133
+ raise "Cohort analysis not supported" unless supports_cohort_analysis?
134
+
135
+ cohort_column = statement =~ /\bcohort_time\b/ ? "cohort_time" : "conversion_time"
136
+ tzname = Blazer.time_zone.tzinfo.name
137
+
138
+ if mysql?
139
+ time_sql = "CONVERT_TZ(cohorts.cohort_time, '+00:00', ?)"
140
+ case period
141
+ when "day"
142
+ date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-%d') AS DATE)"
143
+ date_params = [tzname]
144
+ when "week"
145
+ date_sql = "CAST(DATE_FORMAT(#{time_sql} - INTERVAL ((5 + DAYOFWEEK(#{time_sql})) % 7) DAY, '%Y-%m-%d') AS DATE)"
146
+ date_params = [tzname, tzname]
147
+ else
148
+ date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-01') AS DATE)"
149
+ date_params = [tzname]
150
+ end
151
+ bucket_sql = "CAST(CEIL(TIMESTAMPDIFF(SECOND, cohorts.cohort_time, query.conversion_time) / ?) AS SIGNED)"
152
+ else
153
+ date_sql = "date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date"
154
+ date_params = [period, tzname]
155
+ bucket_sql = "CEIL(EXTRACT(EPOCH FROM query.conversion_time - cohorts.cohort_time) / ?)::int"
156
+ end
157
+
158
+ # WITH not an optimization fence in Postgres 12+
159
+ statement = <<~SQL
160
+ WITH query AS (
161
+ {placeholder}
162
+ ),
163
+ cohorts AS (
164
+ SELECT user_id, MIN(#{cohort_column}) AS cohort_time FROM query
165
+ WHERE user_id IS NOT NULL AND #{cohort_column} IS NOT NULL
166
+ GROUP BY 1
167
+ )
168
+ SELECT
169
+ #{date_sql} AS period,
170
+ 0 AS bucket,
171
+ COUNT(DISTINCT cohorts.user_id)
172
+ FROM cohorts GROUP BY 1
173
+ UNION ALL
174
+ SELECT
175
+ #{date_sql} AS period,
176
+ #{bucket_sql} AS bucket,
177
+ COUNT(DISTINCT query.user_id)
178
+ FROM cohorts INNER JOIN query ON query.user_id = cohorts.user_id
179
+ WHERE query.conversion_time IS NOT NULL
180
+ AND query.conversion_time >= cohorts.cohort_time
181
+ #{cohort_column == "conversion_time" ? "AND query.conversion_time != cohorts.cohort_time" : ""}
182
+ GROUP BY 1, 2
183
+ SQL
184
+ params = [statement] + date_params + date_params + [days.to_i * 86400]
185
+ connection_model.send(:sanitize_sql_array, params)
186
+ end
187
+
188
+ def quoting
189
+ ->(value) { connection_model.connection.quote(value) }
190
+ end
191
+
192
+ # Redshift adapter silently ignores binds
193
+ def parameter_binding
194
+ if postgresql? && (ActiveRecord::VERSION::STRING.to_f >= 6.1 || prepared_statements?)
195
+ # Active Record < 6.1 silently ignores binds with Postgres when prepared statements are disabled
196
+ :numeric
197
+ elsif sqlite?
198
+ :numeric
199
+ elsif mysql? && prepared_statements?
200
+ # Active Record silently ignores binds with MySQL when prepared statements are disabled
201
+ :positional
202
+ elsif sqlserver?
203
+ proc do |statement, variables|
204
+ variables.each_with_index do |(k, _), i|
205
+ statement = statement.gsub("{#{k}}", "@#{i} ")
206
+ end
207
+ [statement, variables.values]
208
+ end
209
+ end
210
+ end
211
+
94
212
  protected
95
213
 
96
- def select_all(statement)
214
+ def select_all(statement, params = [])
215
+ statement = connection_model.send(:sanitize_sql_array, [statement] + params) if params.any?
97
216
  connection_model.connection.select_all(statement)
98
217
  end
99
218
 
@@ -114,20 +233,64 @@ module Blazer
114
233
  ["MySQL", "Mysql2", "Mysql2Spatial"].include?(adapter_name)
115
234
  end
116
235
 
236
+ def sqlite?
237
+ ["SQLite"].include?(adapter_name)
238
+ end
239
+
240
+ def sqlserver?
241
+ ["SQLServer", "tinytds", "mssql"].include?(adapter_name)
242
+ end
243
+
244
+ def snowflake?
245
+ data_source.adapter == "snowflake"
246
+ end
247
+
117
248
  def adapter_name
118
- connection_model.connection.adapter_name
249
+ # prevent bad data source from taking down queries/new
250
+ connection_model.connection.adapter_name rescue nil
251
+ end
252
+
253
+ def default_schema
254
+ @default_schema ||= begin
255
+ if postgresql? || redshift?
256
+ "public"
257
+ elsif sqlserver?
258
+ "dbo"
259
+ elsif connection_model.respond_to?(:connection_db_config)
260
+ connection_model.connection_db_config.database
261
+ else
262
+ connection_model.connection_config[:database]
263
+ end
264
+ end
119
265
  end
120
266
 
121
- def schemas
122
- default_schema = (postgresql? || redshift?) ? "public" : connection_model.connection_config[:database]
123
- settings["schemas"] || [connection_model.connection_config[:schema] || default_schema]
267
+ def add_schemas(query)
268
+ if settings["schemas"]
269
+ where = "table_schema IN (?)"
270
+ schemas = settings["schemas"]
271
+ elsif mysql?
272
+ where = "table_schema IN (?)"
273
+ schemas = [default_schema]
274
+ else
275
+ where = "table_schema NOT IN (?)"
276
+ schemas = ["information_schema"]
277
+ schemas.map!(&:upcase) if snowflake?
278
+ schemas << "pg_catalog" if postgresql? || redshift?
279
+ end
280
+ connection_model.send(:sanitize_sql_array, ["#{query} WHERE #{where}", schemas])
124
281
  end
125
282
 
126
283
  def set_timeout(timeout)
127
284
  if postgresql? || redshift?
128
285
  execute("SET #{use_transaction? ? "LOCAL " : ""}statement_timeout = #{timeout.to_i * 1000}")
129
286
  elsif mysql?
130
- execute("SET max_execution_time = #{timeout.to_i * 1000}")
287
+ # use send as this method is private in Rails 4.2
288
+ mariadb = connection_model.connection.send(:mariadb?) rescue false
289
+ if mariadb
290
+ execute("SET max_statement_time = #{timeout.to_i * 1000}")
291
+ else
292
+ execute("SET max_execution_time = #{timeout.to_i * 1000}")
293
+ end
131
294
  else
132
295
  raise Blazer::TimeoutNotSupported, "Timeout not supported for #{adapter_name} adapter"
133
296
  end
@@ -149,6 +312,10 @@ module Blazer
149
312
  end
150
313
  end
151
314
  end
315
+
316
+ def prepared_statements?
317
+ connection_model.connection.prepared_statements
318
+ end
152
319
  end
153
320
  end
154
321
  end
File without changes
@@ -4,31 +4,13 @@ module Blazer
4
4
  class DataSource
5
5
  extend Forwardable
6
6
 
7
- attr_reader :id, :settings, :adapter, :adapter_instance
7
+ attr_reader :id, :settings
8
8
 
9
- def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel
9
+ def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel, :supports_cohort_analysis?, :cohort_analysis_statement
10
10
 
11
11
  def initialize(id, settings)
12
12
  @id = id
13
13
  @settings = settings
14
-
15
- unless settings["url"] || Rails.env.development?
16
- raise Blazer::Error, "Empty url for data source: #{id}"
17
- end
18
-
19
- @adapter_instance =
20
- case adapter
21
- when "elasticsearch"
22
- Blazer::Adapters::ElasticsearchAdapter.new(self)
23
- when "mongodb"
24
- Blazer::Adapters::MongodbAdapter.new(self)
25
- when "presto"
26
- Blazer::Adapters::PrestoAdapter.new(self)
27
- when "sql"
28
- Blazer::Adapters::SqlAdapter.new(self)
29
- else
30
- raise Blazer::Error, "Unknown adapter"
31
- end
32
14
  end
33
15
 
34
16
  def adapter
@@ -107,8 +89,19 @@ module Blazer
107
89
  Blazer.cache.delete(run_cache_key(run_id))
108
90
  end
109
91
 
92
+ def sub_variables(statement, vars)
93
+ statement = statement.dup
94
+ vars.each do |var, value|
95
+ # use block form to disable back-references
96
+ statement.gsub!("{#{var}}") { quote(value) }
97
+ end
98
+ statement
99
+ end
100
+
110
101
  def run_statement(statement, options = {})
111
- run_id = options[:run_id]
102
+ statement = Statement.new(statement, self) if statement.is_a?(String)
103
+ statement.bind unless statement.bind_statement
104
+
112
105
  async = options[:async]
113
106
  result = nil
114
107
  if cache_mode != "off"
@@ -137,7 +130,7 @@ module Blazer
137
130
  if options[:run_id]
138
131
  comment << ",run_id:#{options[:run_id]}"
139
132
  end
140
- result = run_statement_helper(statement, comment, async ? options[:run_id] : nil)
133
+ result = run_statement_helper(statement, comment, async ? options[:run_id] : nil, options)
141
134
  end
142
135
 
143
136
  result
@@ -152,18 +145,101 @@ module Blazer
152
145
  end
153
146
 
154
147
  def statement_cache_key(statement)
155
- cache_key(["statement", id, Digest::MD5.hexdigest(statement.to_s.gsub("\r\n", "\n"))])
148
+ cache_key(["statement", id, Digest::MD5.hexdigest(statement.bind_statement.to_s.gsub("\r\n", "\n") + statement.bind_values.to_json)])
156
149
  end
157
150
 
158
151
  def run_cache_key(run_id)
159
152
  cache_key(["run", run_id])
160
153
  end
161
154
 
155
+ def quote(value)
156
+ if quoting == :backslash_escape || quoting == :single_quote_escape
157
+ # only need to support types generated by process_vars
158
+ if value.is_a?(Integer) || value.is_a?(Float)
159
+ value.to_s
160
+ elsif value.nil?
161
+ "NULL"
162
+ else
163
+ value = value.to_formatted_s(:db) if value.is_a?(ActiveSupport::TimeWithZone)
164
+
165
+ if quoting == :backslash_escape
166
+ "'#{value.gsub("\\") { "\\\\" }.gsub("'") { "\\'" }}'"
167
+ else
168
+ "'#{value.gsub("'", "''")}'"
169
+ end
170
+ end
171
+ elsif quoting.respond_to?(:call)
172
+ quoting.call(value)
173
+ elsif quoting.nil?
174
+ raise Blazer::Error, "Quoting not specified"
175
+ else
176
+ raise Blazer::Error, "Unknown quoting"
177
+ end
178
+ end
179
+
180
+ def bind_params(statement, variables)
181
+ if parameter_binding == :positional
182
+ locations = []
183
+ variables.each do |k, v|
184
+ i = 0
185
+ while (idx = statement.index("{#{k}}", i))
186
+ locations << [v, idx]
187
+ i = idx + 1
188
+ end
189
+ end
190
+ variables.each do |k, v|
191
+ statement = statement.gsub("{#{k}}", "?")
192
+ end
193
+ [statement, locations.sort_by(&:last).map(&:first)]
194
+ elsif parameter_binding == :numeric
195
+ variables.each_with_index do |(k, v), i|
196
+ # add trailing space if followed by digit
197
+ # try to keep minimal to avoid fixing invalid queries like SELECT{var}
198
+ statement = statement.gsub(/#{Regexp.escape("{#{k}}")}(\d)/, "$#{i + 1} \\1").gsub("{#{k}}", "$#{i + 1}")
199
+ end
200
+ [statement, variables.values]
201
+ elsif parameter_binding.respond_to?(:call)
202
+ parameter_binding.call(statement, variables)
203
+ elsif parameter_binding.nil?
204
+ [sub_variables(statement, variables), []]
205
+ else
206
+ raise Blazer::Error, "Unknown bind parameters"
207
+ end
208
+ end
209
+
162
210
  protected
163
211
 
164
- def run_statement_helper(statement, comment, run_id)
212
+ def adapter_instance
213
+ @adapter_instance ||= begin
214
+ # TODO add required settings to adapters
215
+ unless settings["url"] || Rails.env.development? || ["bigquery", "athena", "snowflake", "salesforce"].include?(settings["adapter"])
216
+ raise Blazer::Error, "Empty url for data source: #{id}"
217
+ end
218
+
219
+ unless Blazer.adapters[adapter]
220
+ raise Blazer::Error, "Unknown adapter"
221
+ end
222
+
223
+ Blazer.adapters[adapter].new(self)
224
+ end
225
+ end
226
+
227
+ def quoting
228
+ @quoting ||= adapter_instance.quoting
229
+ end
230
+
231
+ def parameter_binding
232
+ @parameter_binding ||= adapter_instance.parameter_binding
233
+ end
234
+
235
+ def run_statement_helper(statement, comment, run_id, options)
165
236
  start_time = Time.now
166
- columns, rows, error = @adapter_instance.run_statement(statement, comment)
237
+ columns, rows, error =
238
+ if adapter_instance.parameter_binding
239
+ adapter_instance.run_statement(statement.bind_statement, comment, statement.bind_values)
240
+ else
241
+ adapter_instance.run_statement(statement.bind_statement, comment)
242
+ end
167
243
  duration = Time.now - start_time
168
244
 
169
245
  cache_data = nil
@@ -172,7 +248,7 @@ module Blazer
172
248
  cache_data = Marshal.dump([columns, rows, error, cache ? Time.now : nil]) rescue nil
173
249
  end
174
250
 
175
- if cache && cache_data && @adapter_instance.cachable?(statement)
251
+ if cache && cache_data && adapter_instance.cachable?(statement.bind_statement)
176
252
  Blazer.cache.write(statement_cache_key(statement), cache_data, expires_in: cache_expires_in.to_f * 60)
177
253
  end
178
254
 
@@ -187,11 +263,12 @@ module Blazer
187
263
  Blazer::Result.new(self, columns, rows, error, nil, cache && !cache_data.nil?)
188
264
  end
189
265
 
266
+ # TODO check for adapter with same name, default to sql
190
267
  def detect_adapter
191
- schema = settings["url"].to_s.split("://").first
192
- case schema
193
- when "mongodb", "presto"
194
- schema
268
+ scheme = settings["url"].to_s.split("://").first
269
+ case scheme
270
+ when "mongodb", "presto", "cassandra", "ignite"
271
+ scheme
195
272
  else
196
273
  "sql"
197
274
  end
data/lib/blazer/engine.rb CHANGED
@@ -3,40 +3,38 @@ module Blazer
3
3
  isolate_namespace Blazer
4
4
 
5
5
  initializer "blazer" do |app|
6
- # use a proc instead of a string
7
- app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/application\.(js|css)\z/ }
8
- app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/.+\.(eot|svg|ttf|woff)\z/ }
6
+ if defined?(Sprockets) && Sprockets::VERSION >= "4"
7
+ app.config.assets.precompile << "blazer/application.js"
8
+ app.config.assets.precompile << "blazer/application.css"
9
+ app.config.assets.precompile << "blazer/glyphicons-halflings-regular.eot"
10
+ app.config.assets.precompile << "blazer/glyphicons-halflings-regular.svg"
11
+ app.config.assets.precompile << "blazer/glyphicons-halflings-regular.ttf"
12
+ app.config.assets.precompile << "blazer/glyphicons-halflings-regular.woff"
13
+ app.config.assets.precompile << "blazer/glyphicons-halflings-regular.woff2"
14
+ app.config.assets.precompile << "blazer/favicon.png"
15
+ else
16
+ # use a proc instead of a string
17
+ app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/application\.(js|css)\z/ }
18
+ app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/.+\.(eot|svg|ttf|woff|woff2)\z/ }
19
+ app.config.assets.precompile << proc { |path| path == "blazer/favicon.png" }
20
+ end
9
21
 
10
22
  Blazer.time_zone ||= Blazer.settings["time_zone"] || Time.zone
11
23
  Blazer.audit = Blazer.settings.key?("audit") ? Blazer.settings["audit"] : true
12
24
  Blazer.user_name = Blazer.settings["user_name"] if Blazer.settings["user_name"]
13
25
  Blazer.from_email = Blazer.settings["from_email"] if Blazer.settings["from_email"]
14
- Blazer.before_action = Blazer.settings["before_action"] if Blazer.settings["before_action"]
15
-
16
- Blazer.user_class ||= Blazer.settings.key?("user_class") ? Blazer.settings["user_class"] : (User rescue nil)
17
- Blazer.user_method = Blazer.settings["user_method"]
18
- if Blazer.user_class
19
- Blazer.user_method ||= "current_#{Blazer.user_class.to_s.downcase.singularize}"
20
- end
21
-
26
+ Blazer.before_action = Blazer.settings["before_action_method"] if Blazer.settings["before_action_method"]
22
27
  Blazer.check_schedules = Blazer.settings["check_schedules"] if Blazer.settings.key?("check_schedules")
23
-
24
- if Blazer.user_class
25
- options = Blazer::BELONGS_TO_OPTIONAL.merge(class_name: Blazer.user_class.to_s)
26
- Blazer::Query.belongs_to :creator, options
27
- Blazer::Dashboard.belongs_to :creator, options
28
- Blazer::Check.belongs_to :creator, options
29
- end
30
-
31
28
  Blazer.cache ||= Rails.cache
32
29
 
33
30
  Blazer.anomaly_checks = Blazer.settings["anomaly_checks"] || false
31
+ Blazer.forecasting = Blazer.settings["forecasting"] || false
34
32
  Blazer.async = Blazer.settings["async"] || false
35
- if Blazer.async
36
- require "blazer/run_statement_job"
37
- end
38
-
39
33
  Blazer.images = Blazer.settings["images"] || false
34
+ Blazer.override_csp = Blazer.settings["override_csp"] || false
35
+ Blazer.slack_oauth_token = Blazer.settings["slack_oauth_token"] || ENV["BLAZER_SLACK_OAUTH_TOKEN"]
36
+ Blazer.slack_webhook_url = Blazer.settings["slack_webhook_url"] || ENV["BLAZER_SLACK_WEBHOOK_URL"]
37
+ Blazer.mapbox_access_token = Blazer.settings["mapbox_access_token"] || ENV["MAPBOX_ACCESS_TOKEN"]
40
38
  end
41
39
  end
42
40
  end