blazer 1.7.7 → 2.6.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +242 -33
- data/CONTRIBUTING.md +42 -0
- data/LICENSE.txt +1 -1
- data/README.md +621 -211
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.eot +0 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.svg +0 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.ttf +0 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff +0 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff2 +0 -0
- data/app/assets/images/blazer/favicon.png +0 -0
- data/app/assets/javascripts/blazer/Chart.js +15658 -10011
- data/app/assets/javascripts/blazer/Sortable.js +3413 -848
- data/app/assets/javascripts/blazer/ace/ace.js +21294 -4
- data/app/assets/javascripts/blazer/ace/ext-language_tools.js +1991 -3
- data/app/assets/javascripts/blazer/ace/mode-sql.js +110 -1
- data/app/assets/javascripts/blazer/ace/snippets/sql.js +40 -1
- data/app/assets/javascripts/blazer/ace/snippets/text.js +14 -1
- data/app/assets/javascripts/blazer/ace/theme-twilight.js +116 -1
- data/app/assets/javascripts/blazer/application.js +5 -3
- data/app/assets/javascripts/blazer/bootstrap.js +842 -628
- data/app/assets/javascripts/blazer/chartkick.js +2015 -1244
- data/app/assets/javascripts/blazer/daterangepicker.js +372 -299
- data/app/assets/javascripts/blazer/highlight.min.js +3 -0
- data/app/assets/javascripts/blazer/{jquery_ujs.js → jquery-ujs.js} +161 -75
- data/app/assets/javascripts/blazer/jquery.js +10126 -9562
- data/app/assets/javascripts/blazer/jquery.stickytableheaders.js +321 -259
- data/app/assets/javascripts/blazer/moment-timezone-with-data.js +1546 -0
- data/app/assets/javascripts/blazer/moment.js +5085 -2460
- data/app/assets/javascripts/blazer/queries.js +18 -4
- data/app/assets/javascripts/blazer/routes.js +3 -0
- data/app/assets/javascripts/blazer/selectize.js +3828 -3604
- data/app/assets/javascripts/blazer/stupidtable-custom-settings.js +13 -0
- data/app/assets/javascripts/blazer/stupidtable.js +254 -87
- data/app/assets/javascripts/blazer/vue.js +11175 -6676
- data/app/assets/stylesheets/blazer/application.css +51 -6
- data/app/assets/stylesheets/blazer/bootstrap-propshaft.css +10 -0
- data/app/assets/stylesheets/blazer/bootstrap-sprockets.css.erb +10 -0
- data/app/assets/stylesheets/blazer/{bootstrap.css.erb → bootstrap.css} +1337 -711
- data/app/assets/stylesheets/blazer/{daterangepicker-bs3.css → daterangepicker.css} +207 -172
- data/app/assets/stylesheets/blazer/{selectize.default.css → selectize.css} +26 -10
- data/app/controllers/blazer/base_controller.rb +73 -46
- data/app/controllers/blazer/checks_controller.rb +1 -1
- data/app/controllers/blazer/dashboards_controller.rb +7 -13
- data/app/controllers/blazer/queries_controller.rb +171 -51
- data/app/controllers/blazer/uploads_controller.rb +147 -0
- data/app/helpers/blazer/base_helper.rb +6 -16
- data/app/models/blazer/audit.rb +3 -3
- data/app/models/blazer/check.rb +31 -5
- data/app/models/blazer/dashboard.rb +6 -2
- data/app/models/blazer/dashboard_query.rb +1 -1
- data/app/models/blazer/query.rb +30 -4
- data/app/models/blazer/record.rb +5 -0
- data/app/models/blazer/upload.rb +11 -0
- data/app/models/blazer/uploads_connection.rb +7 -0
- data/app/views/blazer/_nav.html.erb +3 -1
- data/app/views/blazer/_variables.html.erb +48 -23
- data/app/views/blazer/check_mailer/failing_checks.html.erb +1 -0
- data/app/views/blazer/check_mailer/state_change.html.erb +1 -0
- data/app/views/blazer/checks/_form.html.erb +17 -9
- data/app/views/blazer/checks/edit.html.erb +2 -0
- data/app/views/blazer/checks/index.html.erb +37 -5
- data/app/views/blazer/checks/new.html.erb +2 -0
- data/app/views/blazer/dashboards/_form.html.erb +5 -5
- data/app/views/blazer/dashboards/edit.html.erb +2 -0
- data/app/views/blazer/dashboards/new.html.erb +2 -0
- data/app/views/blazer/dashboards/show.html.erb +13 -7
- data/app/views/blazer/queries/_caching.html.erb +16 -0
- data/app/views/blazer/queries/_cohorts.html.erb +48 -0
- data/app/views/blazer/queries/_form.html.erb +23 -13
- data/app/views/blazer/queries/docs.html.erb +137 -0
- data/app/views/blazer/queries/home.html.erb +21 -7
- data/app/views/blazer/queries/run.html.erb +64 -29
- data/app/views/blazer/queries/schema.html.erb +44 -7
- data/app/views/blazer/queries/show.html.erb +15 -8
- data/app/views/blazer/uploads/_form.html.erb +27 -0
- data/app/views/blazer/uploads/edit.html.erb +3 -0
- data/app/views/blazer/uploads/index.html.erb +55 -0
- data/app/views/blazer/uploads/new.html.erb +3 -0
- data/app/views/layouts/blazer/application.html.erb +10 -5
- data/config/routes.rb +10 -1
- data/lib/blazer/adapters/athena_adapter.rb +182 -0
- data/lib/blazer/adapters/base_adapter.rb +24 -1
- data/lib/blazer/adapters/bigquery_adapter.rb +79 -0
- data/lib/blazer/adapters/cassandra_adapter.rb +70 -0
- data/lib/blazer/adapters/drill_adapter.rb +38 -0
- data/lib/blazer/adapters/druid_adapter.rb +102 -0
- data/lib/blazer/adapters/elasticsearch_adapter.rb +30 -18
- data/lib/blazer/adapters/hive_adapter.rb +55 -0
- data/lib/blazer/adapters/ignite_adapter.rb +64 -0
- data/lib/blazer/adapters/influxdb_adapter.rb +57 -0
- data/lib/blazer/adapters/mongodb_adapter.rb +5 -1
- data/lib/blazer/adapters/neo4j_adapter.rb +62 -0
- data/lib/blazer/adapters/opensearch_adapter.rb +52 -0
- data/lib/blazer/adapters/presto_adapter.rb +9 -0
- data/lib/blazer/adapters/salesforce_adapter.rb +50 -0
- data/lib/blazer/adapters/snowflake_adapter.rb +82 -0
- data/lib/blazer/adapters/soda_adapter.rb +105 -0
- data/lib/blazer/adapters/spark_adapter.rb +14 -0
- data/lib/blazer/adapters/sql_adapter.rb +187 -20
- data/{app/mailers → lib}/blazer/check_mailer.rb +0 -0
- data/lib/blazer/data_source.rb +107 -30
- data/lib/blazer/engine.rb +21 -23
- data/lib/blazer/result.rb +95 -29
- data/lib/blazer/run_statement.rb +8 -4
- data/lib/blazer/run_statement_job.rb +8 -9
- data/lib/blazer/slack_notifier.rb +94 -0
- data/lib/blazer/statement.rb +75 -0
- data/lib/blazer/version.rb +1 -1
- data/lib/blazer.rb +154 -26
- data/lib/generators/blazer/install_generator.rb +7 -18
- data/lib/generators/blazer/templates/{config.yml → config.yml.tt} +26 -3
- data/lib/generators/blazer/templates/{install.rb → install.rb.tt} +6 -4
- data/lib/generators/blazer/templates/uploads.rb.tt +10 -0
- data/lib/generators/blazer/uploads_generator.rb +18 -0
- data/lib/tasks/blazer.rake +11 -1
- data/licenses/LICENSE-ace.txt +24 -0
- data/licenses/LICENSE-bootstrap.txt +21 -0
- data/licenses/LICENSE-chart.js.txt +9 -0
- data/licenses/LICENSE-chartkick.js.txt +22 -0
- data/licenses/LICENSE-daterangepicker.txt +21 -0
- data/licenses/LICENSE-fuzzysearch.txt +20 -0
- data/licenses/LICENSE-highlight.js.txt +29 -0
- data/licenses/LICENSE-jquery-ujs.txt +20 -0
- data/licenses/LICENSE-jquery.txt +20 -0
- data/licenses/LICENSE-moment-timezone.txt +20 -0
- data/licenses/LICENSE-moment.txt +22 -0
- data/licenses/LICENSE-selectize.txt +202 -0
- data/licenses/LICENSE-sortable.txt +21 -0
- data/licenses/LICENSE-stickytableheaders.txt +20 -0
- data/licenses/LICENSE-stupidtable.txt +19 -0
- data/licenses/LICENSE-vue.txt +21 -0
- metadata +83 -53
- data/.gitignore +0 -14
- data/Gemfile +0 -4
- data/Rakefile +0 -1
- data/app/assets/javascripts/blazer/highlight.pack.js +0 -1
- data/app/assets/javascripts/blazer/moment-timezone.js +0 -1007
- data/blazer.gemspec +0 -26
@@ -0,0 +1,105 @@
|
|
1
|
+
module Blazer
|
2
|
+
module Adapters
|
3
|
+
class SodaAdapter < BaseAdapter
|
4
|
+
def run_statement(statement, comment)
|
5
|
+
require "json"
|
6
|
+
require "net/http"
|
7
|
+
require "uri"
|
8
|
+
|
9
|
+
columns = []
|
10
|
+
rows = []
|
11
|
+
error = nil
|
12
|
+
|
13
|
+
# remove comments manually
|
14
|
+
statement = statement.gsub(/--.+/, "")
|
15
|
+
# only supports single line /* */ comments
|
16
|
+
# regex not perfect, but should be good enough
|
17
|
+
statement = statement.gsub(/\/\*.+\*\//, "")
|
18
|
+
|
19
|
+
# remove trailing semicolon
|
20
|
+
statement = statement.sub(/;\s*\z/, "")
|
21
|
+
|
22
|
+
# remove whitespace
|
23
|
+
statement = statement.squish
|
24
|
+
|
25
|
+
uri = URI(settings["url"])
|
26
|
+
uri.query = URI.encode_www_form("$query" => statement)
|
27
|
+
|
28
|
+
req = Net::HTTP::Get.new(uri)
|
29
|
+
req["X-App-Token"] = settings["app_token"] if settings["app_token"]
|
30
|
+
|
31
|
+
options = {
|
32
|
+
use_ssl: uri.scheme == "https",
|
33
|
+
open_timeout: 3,
|
34
|
+
read_timeout: 30
|
35
|
+
}
|
36
|
+
|
37
|
+
begin
|
38
|
+
# use Net::HTTP instead of soda-ruby for types and better error messages
|
39
|
+
res = Net::HTTP.start(uri.hostname, uri.port, options) do |http|
|
40
|
+
http.request(req)
|
41
|
+
end
|
42
|
+
|
43
|
+
if res.is_a?(Net::HTTPSuccess)
|
44
|
+
body = JSON.parse(res.body)
|
45
|
+
|
46
|
+
columns = JSON.parse(res["x-soda2-fields"])
|
47
|
+
column_types = columns.zip(JSON.parse(res["x-soda2-types"])).to_h
|
48
|
+
|
49
|
+
columns.reject! { |f| f.start_with?(":@") }
|
50
|
+
# rows can be missing some keys in JSON, so need to map by column
|
51
|
+
rows = body.map { |r| columns.map { |c| r[c] } }
|
52
|
+
|
53
|
+
columns.each_with_index do |column, i|
|
54
|
+
# nothing to do for boolean
|
55
|
+
case column_types[column]
|
56
|
+
when "number"
|
57
|
+
# check if likely an integer column
|
58
|
+
if rows.all? { |r| r[i].to_i == r[i].to_f }
|
59
|
+
rows.each do |row|
|
60
|
+
row[i] = row[i].to_i
|
61
|
+
end
|
62
|
+
else
|
63
|
+
rows.each do |row|
|
64
|
+
row[i] = row[i].to_f
|
65
|
+
end
|
66
|
+
end
|
67
|
+
when "floating_timestamp"
|
68
|
+
# check if likely a date column
|
69
|
+
if rows.all? { |r| r[i].end_with?("T00:00:00.000") }
|
70
|
+
rows.each do |row|
|
71
|
+
row[i] = Date.parse(row[i])
|
72
|
+
end
|
73
|
+
else
|
74
|
+
utc = ActiveSupport::TimeZone["Etc/UTC"]
|
75
|
+
rows.each do |row|
|
76
|
+
row[i] = utc.parse(row[i])
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
else
|
82
|
+
error = JSON.parse(res.body)["message"] rescue "Bad response: #{res.code}"
|
83
|
+
end
|
84
|
+
rescue => e
|
85
|
+
error = e.message
|
86
|
+
end
|
87
|
+
|
88
|
+
[columns, rows, error]
|
89
|
+
end
|
90
|
+
|
91
|
+
def preview_statement
|
92
|
+
"SELECT * LIMIT 10"
|
93
|
+
end
|
94
|
+
|
95
|
+
def tables
|
96
|
+
["all"]
|
97
|
+
end
|
98
|
+
|
99
|
+
# https://dev.socrata.com/docs/datatypes/text.html
|
100
|
+
def quoting
|
101
|
+
:single_quote_escape
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Blazer
|
2
|
+
module Adapters
|
3
|
+
class SparkAdapter < HiveAdapter
|
4
|
+
def tables
|
5
|
+
client.execute("SHOW TABLES").map { |r| r["tableName"] }
|
6
|
+
end
|
7
|
+
|
8
|
+
# https://spark.apache.org/docs/latest/sql-ref-literals.html
|
9
|
+
def quoting
|
10
|
+
:backslash_escape
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -15,7 +15,7 @@ module Blazer
|
|
15
15
|
end
|
16
16
|
end
|
17
17
|
|
18
|
-
def run_statement(statement, comment)
|
18
|
+
def run_statement(statement, comment, bind_params = [])
|
19
19
|
columns = []
|
20
20
|
rows = []
|
21
21
|
error = nil
|
@@ -24,16 +24,17 @@ module Blazer
|
|
24
24
|
in_transaction do
|
25
25
|
set_timeout(data_source.timeout) if data_source.timeout
|
26
26
|
|
27
|
-
|
27
|
+
binds = bind_params.map { |v| ActiveRecord::Relation::QueryAttribute.new(nil, v, ActiveRecord::Type::Value.new) }
|
28
|
+
result = connection_model.connection.select_all("#{statement} /*#{comment}*/", nil, binds)
|
28
29
|
columns = result.columns
|
29
|
-
cast_method = Rails::VERSION::MAJOR < 5 ? :type_cast : :cast_value
|
30
30
|
result.rows.each do |untyped_row|
|
31
|
-
rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] ? result.column_types[c].send(
|
31
|
+
rows << (result.column_types.empty? ? untyped_row : columns.each_with_index.map { |c, i| untyped_row[i] && result.column_types[c] ? result.column_types[c].send(:cast_value, untyped_row[i]) : untyped_row[i] })
|
32
32
|
end
|
33
33
|
end
|
34
34
|
rescue => e
|
35
35
|
error = e.message.sub(/.+ERROR: /, "")
|
36
36
|
error = Blazer::TIMEOUT_MESSAGE if Blazer::TIMEOUT_ERRORS.any? { |e| error.include?(e) }
|
37
|
+
error = Blazer::VARIABLE_MESSAGE if error.include?("syntax error at or near \"$") || error.include?("Incorrect syntax near '@") || error.include?("your MySQL server version for the right syntax to use near '?")
|
37
38
|
reconnect if error.include?("PG::ConnectionBad")
|
38
39
|
end
|
39
40
|
|
@@ -41,18 +42,38 @@ module Blazer
|
|
41
42
|
end
|
42
43
|
|
43
44
|
def tables
|
44
|
-
|
45
|
-
result.
|
45
|
+
sql = add_schemas("SELECT table_schema, table_name FROM information_schema.tables")
|
46
|
+
result = data_source.run_statement(sql, refresh_cache: true)
|
47
|
+
if postgresql? || redshift? || snowflake?
|
48
|
+
result.rows.sort_by { |r| [r[0] == default_schema ? "" : r[0], r[1]] }.map do |row|
|
49
|
+
table =
|
50
|
+
if row[0] == default_schema
|
51
|
+
row[1]
|
52
|
+
else
|
53
|
+
"#{row[0]}.#{row[1]}"
|
54
|
+
end
|
55
|
+
|
56
|
+
table = table.downcase if snowflake?
|
57
|
+
|
58
|
+
{
|
59
|
+
table: table,
|
60
|
+
value: connection_model.connection.quote_table_name(table)
|
61
|
+
}
|
62
|
+
end
|
63
|
+
else
|
64
|
+
result.rows.map(&:second).sort
|
65
|
+
end
|
46
66
|
end
|
47
67
|
|
48
68
|
def schema
|
49
|
-
|
50
|
-
result
|
69
|
+
sql = add_schemas("SELECT table_schema, table_name, column_name, data_type, ordinal_position FROM information_schema.columns")
|
70
|
+
result = data_source.run_statement(sql)
|
71
|
+
result.rows.group_by { |r| [r[0], r[1]] }.map { |k, vs| {schema: k[0], table: k[1], columns: vs.sort_by { |v| v[2] }.map { |v| {name: v[2], data_type: v[3]} }} }.sort_by { |t| [t[:schema] == default_schema ? "" : t[:schema], t[:table]] }
|
51
72
|
end
|
52
73
|
|
53
74
|
def preview_statement
|
54
|
-
if
|
55
|
-
"SELECT * FROM
|
75
|
+
if sqlserver?
|
76
|
+
"SELECT TOP (10) * FROM {table}"
|
56
77
|
else
|
57
78
|
"SELECT * FROM {table} LIMIT 10"
|
58
79
|
end
|
@@ -64,13 +85,25 @@ module Blazer
|
|
64
85
|
|
65
86
|
def cost(statement)
|
66
87
|
result = explain(statement)
|
67
|
-
|
68
|
-
|
88
|
+
if sqlserver?
|
89
|
+
result["TotalSubtreeCost"]
|
90
|
+
else
|
91
|
+
match = /cost=\d+\.\d+..(\d+\.\d+) /.match(result)
|
92
|
+
match[1] if match
|
93
|
+
end
|
69
94
|
end
|
70
95
|
|
71
96
|
def explain(statement)
|
72
97
|
if postgresql? || redshift?
|
73
98
|
select_all("EXPLAIN #{statement}").rows.first.first
|
99
|
+
elsif sqlserver?
|
100
|
+
begin
|
101
|
+
execute("SET SHOWPLAN_ALL ON")
|
102
|
+
result = select_all(statement).each.first
|
103
|
+
ensure
|
104
|
+
execute("SET SHOWPLAN_ALL OFF")
|
105
|
+
end
|
106
|
+
result
|
74
107
|
end
|
75
108
|
rescue
|
76
109
|
nil
|
@@ -78,9 +111,9 @@ module Blazer
|
|
78
111
|
|
79
112
|
def cancel(run_id)
|
80
113
|
if postgresql?
|
81
|
-
select_all("SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND query LIKE
|
114
|
+
select_all("SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND query LIKE ?", ["%,run_id:#{run_id}%"])
|
82
115
|
elsif redshift?
|
83
|
-
first_row = select_all("SELECT pid FROM stv_recents WHERE status = 'Running' AND query LIKE
|
116
|
+
first_row = select_all("SELECT pid FROM stv_recents WHERE status = 'Running' AND query LIKE ?", ["%,run_id:#{run_id}%"]).first
|
84
117
|
if first_row
|
85
118
|
select_all("CANCEL #{first_row["pid"].to_i}")
|
86
119
|
end
|
@@ -91,9 +124,95 @@ module Blazer
|
|
91
124
|
!%w[CREATE ALTER UPDATE INSERT DELETE].include?(statement.split.first.to_s.upcase)
|
92
125
|
end
|
93
126
|
|
127
|
+
def supports_cohort_analysis?
|
128
|
+
postgresql? || mysql?
|
129
|
+
end
|
130
|
+
|
131
|
+
# TODO treat date columns as already in time zone
|
132
|
+
def cohort_analysis_statement(statement, period:, days:)
|
133
|
+
raise "Cohort analysis not supported" unless supports_cohort_analysis?
|
134
|
+
|
135
|
+
cohort_column = statement =~ /\bcohort_time\b/ ? "cohort_time" : "conversion_time"
|
136
|
+
tzname = Blazer.time_zone.tzinfo.name
|
137
|
+
|
138
|
+
if mysql?
|
139
|
+
time_sql = "CONVERT_TZ(cohorts.cohort_time, '+00:00', ?)"
|
140
|
+
case period
|
141
|
+
when "day"
|
142
|
+
date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-%d') AS DATE)"
|
143
|
+
date_params = [tzname]
|
144
|
+
when "week"
|
145
|
+
date_sql = "CAST(DATE_FORMAT(#{time_sql} - INTERVAL ((5 + DAYOFWEEK(#{time_sql})) % 7) DAY, '%Y-%m-%d') AS DATE)"
|
146
|
+
date_params = [tzname, tzname]
|
147
|
+
else
|
148
|
+
date_sql = "CAST(DATE_FORMAT(#{time_sql}, '%Y-%m-01') AS DATE)"
|
149
|
+
date_params = [tzname]
|
150
|
+
end
|
151
|
+
bucket_sql = "CAST(CEIL(TIMESTAMPDIFF(SECOND, cohorts.cohort_time, query.conversion_time) / ?) AS SIGNED)"
|
152
|
+
else
|
153
|
+
date_sql = "date_trunc(?, cohorts.cohort_time::timestamptz AT TIME ZONE ?)::date"
|
154
|
+
date_params = [period, tzname]
|
155
|
+
bucket_sql = "CEIL(EXTRACT(EPOCH FROM query.conversion_time - cohorts.cohort_time) / ?)::int"
|
156
|
+
end
|
157
|
+
|
158
|
+
# WITH not an optimization fence in Postgres 12+
|
159
|
+
statement = <<~SQL
|
160
|
+
WITH query AS (
|
161
|
+
{placeholder}
|
162
|
+
),
|
163
|
+
cohorts AS (
|
164
|
+
SELECT user_id, MIN(#{cohort_column}) AS cohort_time FROM query
|
165
|
+
WHERE user_id IS NOT NULL AND #{cohort_column} IS NOT NULL
|
166
|
+
GROUP BY 1
|
167
|
+
)
|
168
|
+
SELECT
|
169
|
+
#{date_sql} AS period,
|
170
|
+
0 AS bucket,
|
171
|
+
COUNT(DISTINCT cohorts.user_id)
|
172
|
+
FROM cohorts GROUP BY 1
|
173
|
+
UNION ALL
|
174
|
+
SELECT
|
175
|
+
#{date_sql} AS period,
|
176
|
+
#{bucket_sql} AS bucket,
|
177
|
+
COUNT(DISTINCT query.user_id)
|
178
|
+
FROM cohorts INNER JOIN query ON query.user_id = cohorts.user_id
|
179
|
+
WHERE query.conversion_time IS NOT NULL
|
180
|
+
AND query.conversion_time >= cohorts.cohort_time
|
181
|
+
#{cohort_column == "conversion_time" ? "AND query.conversion_time != cohorts.cohort_time" : ""}
|
182
|
+
GROUP BY 1, 2
|
183
|
+
SQL
|
184
|
+
params = [statement] + date_params + date_params + [days.to_i * 86400]
|
185
|
+
connection_model.send(:sanitize_sql_array, params)
|
186
|
+
end
|
187
|
+
|
188
|
+
def quoting
|
189
|
+
->(value) { connection_model.connection.quote(value) }
|
190
|
+
end
|
191
|
+
|
192
|
+
# Redshift adapter silently ignores binds
|
193
|
+
def parameter_binding
|
194
|
+
if postgresql? && (ActiveRecord::VERSION::STRING.to_f >= 6.1 || prepared_statements?)
|
195
|
+
# Active Record < 6.1 silently ignores binds with Postgres when prepared statements are disabled
|
196
|
+
:numeric
|
197
|
+
elsif sqlite?
|
198
|
+
:numeric
|
199
|
+
elsif mysql? && prepared_statements?
|
200
|
+
# Active Record silently ignores binds with MySQL when prepared statements are disabled
|
201
|
+
:positional
|
202
|
+
elsif sqlserver?
|
203
|
+
proc do |statement, variables|
|
204
|
+
variables.each_with_index do |(k, _), i|
|
205
|
+
statement = statement.gsub("{#{k}}", "@#{i} ")
|
206
|
+
end
|
207
|
+
[statement, variables.values]
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
94
212
|
protected
|
95
213
|
|
96
|
-
def select_all(statement)
|
214
|
+
def select_all(statement, params = [])
|
215
|
+
statement = connection_model.send(:sanitize_sql_array, [statement] + params) if params.any?
|
97
216
|
connection_model.connection.select_all(statement)
|
98
217
|
end
|
99
218
|
|
@@ -114,20 +233,64 @@ module Blazer
|
|
114
233
|
["MySQL", "Mysql2", "Mysql2Spatial"].include?(adapter_name)
|
115
234
|
end
|
116
235
|
|
236
|
+
def sqlite?
|
237
|
+
["SQLite"].include?(adapter_name)
|
238
|
+
end
|
239
|
+
|
240
|
+
def sqlserver?
|
241
|
+
["SQLServer", "tinytds", "mssql"].include?(adapter_name)
|
242
|
+
end
|
243
|
+
|
244
|
+
def snowflake?
|
245
|
+
data_source.adapter == "snowflake"
|
246
|
+
end
|
247
|
+
|
117
248
|
def adapter_name
|
118
|
-
|
249
|
+
# prevent bad data source from taking down queries/new
|
250
|
+
connection_model.connection.adapter_name rescue nil
|
251
|
+
end
|
252
|
+
|
253
|
+
def default_schema
|
254
|
+
@default_schema ||= begin
|
255
|
+
if postgresql? || redshift?
|
256
|
+
"public"
|
257
|
+
elsif sqlserver?
|
258
|
+
"dbo"
|
259
|
+
elsif connection_model.respond_to?(:connection_db_config)
|
260
|
+
connection_model.connection_db_config.database
|
261
|
+
else
|
262
|
+
connection_model.connection_config[:database]
|
263
|
+
end
|
264
|
+
end
|
119
265
|
end
|
120
266
|
|
121
|
-
def
|
122
|
-
|
123
|
-
|
267
|
+
def add_schemas(query)
|
268
|
+
if settings["schemas"]
|
269
|
+
where = "table_schema IN (?)"
|
270
|
+
schemas = settings["schemas"]
|
271
|
+
elsif mysql?
|
272
|
+
where = "table_schema IN (?)"
|
273
|
+
schemas = [default_schema]
|
274
|
+
else
|
275
|
+
where = "table_schema NOT IN (?)"
|
276
|
+
schemas = ["information_schema"]
|
277
|
+
schemas.map!(&:upcase) if snowflake?
|
278
|
+
schemas << "pg_catalog" if postgresql? || redshift?
|
279
|
+
end
|
280
|
+
connection_model.send(:sanitize_sql_array, ["#{query} WHERE #{where}", schemas])
|
124
281
|
end
|
125
282
|
|
126
283
|
def set_timeout(timeout)
|
127
284
|
if postgresql? || redshift?
|
128
285
|
execute("SET #{use_transaction? ? "LOCAL " : ""}statement_timeout = #{timeout.to_i * 1000}")
|
129
286
|
elsif mysql?
|
130
|
-
|
287
|
+
# use send as this method is private in Rails 4.2
|
288
|
+
mariadb = connection_model.connection.send(:mariadb?) rescue false
|
289
|
+
if mariadb
|
290
|
+
execute("SET max_statement_time = #{timeout.to_i * 1000}")
|
291
|
+
else
|
292
|
+
execute("SET max_execution_time = #{timeout.to_i * 1000}")
|
293
|
+
end
|
131
294
|
else
|
132
295
|
raise Blazer::TimeoutNotSupported, "Timeout not supported for #{adapter_name} adapter"
|
133
296
|
end
|
@@ -149,6 +312,10 @@ module Blazer
|
|
149
312
|
end
|
150
313
|
end
|
151
314
|
end
|
315
|
+
|
316
|
+
def prepared_statements?
|
317
|
+
connection_model.connection.prepared_statements
|
318
|
+
end
|
152
319
|
end
|
153
320
|
end
|
154
321
|
end
|
File without changes
|
data/lib/blazer/data_source.rb
CHANGED
@@ -4,31 +4,13 @@ module Blazer
|
|
4
4
|
class DataSource
|
5
5
|
extend Forwardable
|
6
6
|
|
7
|
-
attr_reader :id, :settings
|
7
|
+
attr_reader :id, :settings
|
8
8
|
|
9
|
-
def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel
|
9
|
+
def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel, :supports_cohort_analysis?, :cohort_analysis_statement
|
10
10
|
|
11
11
|
def initialize(id, settings)
|
12
12
|
@id = id
|
13
13
|
@settings = settings
|
14
|
-
|
15
|
-
unless settings["url"] || Rails.env.development?
|
16
|
-
raise Blazer::Error, "Empty url for data source: #{id}"
|
17
|
-
end
|
18
|
-
|
19
|
-
@adapter_instance =
|
20
|
-
case adapter
|
21
|
-
when "elasticsearch"
|
22
|
-
Blazer::Adapters::ElasticsearchAdapter.new(self)
|
23
|
-
when "mongodb"
|
24
|
-
Blazer::Adapters::MongodbAdapter.new(self)
|
25
|
-
when "presto"
|
26
|
-
Blazer::Adapters::PrestoAdapter.new(self)
|
27
|
-
when "sql"
|
28
|
-
Blazer::Adapters::SqlAdapter.new(self)
|
29
|
-
else
|
30
|
-
raise Blazer::Error, "Unknown adapter"
|
31
|
-
end
|
32
14
|
end
|
33
15
|
|
34
16
|
def adapter
|
@@ -107,8 +89,19 @@ module Blazer
|
|
107
89
|
Blazer.cache.delete(run_cache_key(run_id))
|
108
90
|
end
|
109
91
|
|
92
|
+
def sub_variables(statement, vars)
|
93
|
+
statement = statement.dup
|
94
|
+
vars.each do |var, value|
|
95
|
+
# use block form to disable back-references
|
96
|
+
statement.gsub!("{#{var}}") { quote(value) }
|
97
|
+
end
|
98
|
+
statement
|
99
|
+
end
|
100
|
+
|
110
101
|
def run_statement(statement, options = {})
|
111
|
-
|
102
|
+
statement = Statement.new(statement, self) if statement.is_a?(String)
|
103
|
+
statement.bind unless statement.bind_statement
|
104
|
+
|
112
105
|
async = options[:async]
|
113
106
|
result = nil
|
114
107
|
if cache_mode != "off"
|
@@ -137,7 +130,7 @@ module Blazer
|
|
137
130
|
if options[:run_id]
|
138
131
|
comment << ",run_id:#{options[:run_id]}"
|
139
132
|
end
|
140
|
-
result = run_statement_helper(statement, comment, async ? options[:run_id] : nil)
|
133
|
+
result = run_statement_helper(statement, comment, async ? options[:run_id] : nil, options)
|
141
134
|
end
|
142
135
|
|
143
136
|
result
|
@@ -152,18 +145,101 @@ module Blazer
|
|
152
145
|
end
|
153
146
|
|
154
147
|
def statement_cache_key(statement)
|
155
|
-
cache_key(["statement", id, Digest::MD5.hexdigest(statement.to_s.gsub("\r\n", "\n"))])
|
148
|
+
cache_key(["statement", id, Digest::MD5.hexdigest(statement.bind_statement.to_s.gsub("\r\n", "\n") + statement.bind_values.to_json)])
|
156
149
|
end
|
157
150
|
|
158
151
|
def run_cache_key(run_id)
|
159
152
|
cache_key(["run", run_id])
|
160
153
|
end
|
161
154
|
|
155
|
+
def quote(value)
|
156
|
+
if quoting == :backslash_escape || quoting == :single_quote_escape
|
157
|
+
# only need to support types generated by process_vars
|
158
|
+
if value.is_a?(Integer) || value.is_a?(Float)
|
159
|
+
value.to_s
|
160
|
+
elsif value.nil?
|
161
|
+
"NULL"
|
162
|
+
else
|
163
|
+
value = value.to_formatted_s(:db) if value.is_a?(ActiveSupport::TimeWithZone)
|
164
|
+
|
165
|
+
if quoting == :backslash_escape
|
166
|
+
"'#{value.gsub("\\") { "\\\\" }.gsub("'") { "\\'" }}'"
|
167
|
+
else
|
168
|
+
"'#{value.gsub("'", "''")}'"
|
169
|
+
end
|
170
|
+
end
|
171
|
+
elsif quoting.respond_to?(:call)
|
172
|
+
quoting.call(value)
|
173
|
+
elsif quoting.nil?
|
174
|
+
raise Blazer::Error, "Quoting not specified"
|
175
|
+
else
|
176
|
+
raise Blazer::Error, "Unknown quoting"
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
def bind_params(statement, variables)
|
181
|
+
if parameter_binding == :positional
|
182
|
+
locations = []
|
183
|
+
variables.each do |k, v|
|
184
|
+
i = 0
|
185
|
+
while (idx = statement.index("{#{k}}", i))
|
186
|
+
locations << [v, idx]
|
187
|
+
i = idx + 1
|
188
|
+
end
|
189
|
+
end
|
190
|
+
variables.each do |k, v|
|
191
|
+
statement = statement.gsub("{#{k}}", "?")
|
192
|
+
end
|
193
|
+
[statement, locations.sort_by(&:last).map(&:first)]
|
194
|
+
elsif parameter_binding == :numeric
|
195
|
+
variables.each_with_index do |(k, v), i|
|
196
|
+
# add trailing space if followed by digit
|
197
|
+
# try to keep minimal to avoid fixing invalid queries like SELECT{var}
|
198
|
+
statement = statement.gsub(/#{Regexp.escape("{#{k}}")}(\d)/, "$#{i + 1} \\1").gsub("{#{k}}", "$#{i + 1}")
|
199
|
+
end
|
200
|
+
[statement, variables.values]
|
201
|
+
elsif parameter_binding.respond_to?(:call)
|
202
|
+
parameter_binding.call(statement, variables)
|
203
|
+
elsif parameter_binding.nil?
|
204
|
+
[sub_variables(statement, variables), []]
|
205
|
+
else
|
206
|
+
raise Blazer::Error, "Unknown bind parameters"
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
162
210
|
protected
|
163
211
|
|
164
|
-
def
|
212
|
+
def adapter_instance
|
213
|
+
@adapter_instance ||= begin
|
214
|
+
# TODO add required settings to adapters
|
215
|
+
unless settings["url"] || Rails.env.development? || ["bigquery", "athena", "snowflake", "salesforce"].include?(settings["adapter"])
|
216
|
+
raise Blazer::Error, "Empty url for data source: #{id}"
|
217
|
+
end
|
218
|
+
|
219
|
+
unless Blazer.adapters[adapter]
|
220
|
+
raise Blazer::Error, "Unknown adapter"
|
221
|
+
end
|
222
|
+
|
223
|
+
Blazer.adapters[adapter].new(self)
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
def quoting
|
228
|
+
@quoting ||= adapter_instance.quoting
|
229
|
+
end
|
230
|
+
|
231
|
+
def parameter_binding
|
232
|
+
@parameter_binding ||= adapter_instance.parameter_binding
|
233
|
+
end
|
234
|
+
|
235
|
+
def run_statement_helper(statement, comment, run_id, options)
|
165
236
|
start_time = Time.now
|
166
|
-
columns, rows, error =
|
237
|
+
columns, rows, error =
|
238
|
+
if adapter_instance.parameter_binding
|
239
|
+
adapter_instance.run_statement(statement.bind_statement, comment, statement.bind_values)
|
240
|
+
else
|
241
|
+
adapter_instance.run_statement(statement.bind_statement, comment)
|
242
|
+
end
|
167
243
|
duration = Time.now - start_time
|
168
244
|
|
169
245
|
cache_data = nil
|
@@ -172,7 +248,7 @@ module Blazer
|
|
172
248
|
cache_data = Marshal.dump([columns, rows, error, cache ? Time.now : nil]) rescue nil
|
173
249
|
end
|
174
250
|
|
175
|
-
if cache && cache_data &&
|
251
|
+
if cache && cache_data && adapter_instance.cachable?(statement.bind_statement)
|
176
252
|
Blazer.cache.write(statement_cache_key(statement), cache_data, expires_in: cache_expires_in.to_f * 60)
|
177
253
|
end
|
178
254
|
|
@@ -187,11 +263,12 @@ module Blazer
|
|
187
263
|
Blazer::Result.new(self, columns, rows, error, nil, cache && !cache_data.nil?)
|
188
264
|
end
|
189
265
|
|
266
|
+
# TODO check for adapter with same name, default to sql
|
190
267
|
def detect_adapter
|
191
|
-
|
192
|
-
case
|
193
|
-
when "mongodb", "presto"
|
194
|
-
|
268
|
+
scheme = settings["url"].to_s.split("://").first
|
269
|
+
case scheme
|
270
|
+
when "mongodb", "presto", "cassandra", "ignite"
|
271
|
+
scheme
|
195
272
|
else
|
196
273
|
"sql"
|
197
274
|
end
|
data/lib/blazer/engine.rb
CHANGED
@@ -3,40 +3,38 @@ module Blazer
|
|
3
3
|
isolate_namespace Blazer
|
4
4
|
|
5
5
|
initializer "blazer" do |app|
|
6
|
-
|
7
|
-
|
8
|
-
|
6
|
+
if defined?(Sprockets) && Sprockets::VERSION >= "4"
|
7
|
+
app.config.assets.precompile << "blazer/application.js"
|
8
|
+
app.config.assets.precompile << "blazer/application.css"
|
9
|
+
app.config.assets.precompile << "blazer/glyphicons-halflings-regular.eot"
|
10
|
+
app.config.assets.precompile << "blazer/glyphicons-halflings-regular.svg"
|
11
|
+
app.config.assets.precompile << "blazer/glyphicons-halflings-regular.ttf"
|
12
|
+
app.config.assets.precompile << "blazer/glyphicons-halflings-regular.woff"
|
13
|
+
app.config.assets.precompile << "blazer/glyphicons-halflings-regular.woff2"
|
14
|
+
app.config.assets.precompile << "blazer/favicon.png"
|
15
|
+
else
|
16
|
+
# use a proc instead of a string
|
17
|
+
app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/application\.(js|css)\z/ }
|
18
|
+
app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/.+\.(eot|svg|ttf|woff|woff2)\z/ }
|
19
|
+
app.config.assets.precompile << proc { |path| path == "blazer/favicon.png" }
|
20
|
+
end
|
9
21
|
|
10
22
|
Blazer.time_zone ||= Blazer.settings["time_zone"] || Time.zone
|
11
23
|
Blazer.audit = Blazer.settings.key?("audit") ? Blazer.settings["audit"] : true
|
12
24
|
Blazer.user_name = Blazer.settings["user_name"] if Blazer.settings["user_name"]
|
13
25
|
Blazer.from_email = Blazer.settings["from_email"] if Blazer.settings["from_email"]
|
14
|
-
Blazer.before_action = Blazer.settings["
|
15
|
-
|
16
|
-
Blazer.user_class ||= Blazer.settings.key?("user_class") ? Blazer.settings["user_class"] : (User rescue nil)
|
17
|
-
Blazer.user_method = Blazer.settings["user_method"]
|
18
|
-
if Blazer.user_class
|
19
|
-
Blazer.user_method ||= "current_#{Blazer.user_class.to_s.downcase.singularize}"
|
20
|
-
end
|
21
|
-
|
26
|
+
Blazer.before_action = Blazer.settings["before_action_method"] if Blazer.settings["before_action_method"]
|
22
27
|
Blazer.check_schedules = Blazer.settings["check_schedules"] if Blazer.settings.key?("check_schedules")
|
23
|
-
|
24
|
-
if Blazer.user_class
|
25
|
-
options = Blazer::BELONGS_TO_OPTIONAL.merge(class_name: Blazer.user_class.to_s)
|
26
|
-
Blazer::Query.belongs_to :creator, options
|
27
|
-
Blazer::Dashboard.belongs_to :creator, options
|
28
|
-
Blazer::Check.belongs_to :creator, options
|
29
|
-
end
|
30
|
-
|
31
28
|
Blazer.cache ||= Rails.cache
|
32
29
|
|
33
30
|
Blazer.anomaly_checks = Blazer.settings["anomaly_checks"] || false
|
31
|
+
Blazer.forecasting = Blazer.settings["forecasting"] || false
|
34
32
|
Blazer.async = Blazer.settings["async"] || false
|
35
|
-
if Blazer.async
|
36
|
-
require "blazer/run_statement_job"
|
37
|
-
end
|
38
|
-
|
39
33
|
Blazer.images = Blazer.settings["images"] || false
|
34
|
+
Blazer.override_csp = Blazer.settings["override_csp"] || false
|
35
|
+
Blazer.slack_oauth_token = Blazer.settings["slack_oauth_token"] || ENV["BLAZER_SLACK_OAUTH_TOKEN"]
|
36
|
+
Blazer.slack_webhook_url = Blazer.settings["slack_webhook_url"] || ENV["BLAZER_SLACK_WEBHOOK_URL"]
|
37
|
+
Blazer.mapbox_access_token = Blazer.settings["mapbox_access_token"] || ENV["MAPBOX_ACCESS_TOKEN"]
|
40
38
|
end
|
41
39
|
end
|
42
40
|
end
|