sql-jarvis 1.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (98) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +14 -0
  3. data/CHANGELOG.md +228 -0
  4. data/Gemfile +4 -0
  5. data/LICENSE.txt +22 -0
  6. data/README.md +775 -0
  7. data/Rakefile +1 -0
  8. data/app/assets/fonts/blazer/glyphicons-halflings-regular.eot +0 -0
  9. data/app/assets/fonts/blazer/glyphicons-halflings-regular.svg +288 -0
  10. data/app/assets/fonts/blazer/glyphicons-halflings-regular.ttf +0 -0
  11. data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff +0 -0
  12. data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff2 +0 -0
  13. data/app/assets/javascripts/blazer/Chart.js +14145 -0
  14. data/app/assets/javascripts/blazer/Sortable.js +1144 -0
  15. data/app/assets/javascripts/blazer/ace.js +6 -0
  16. data/app/assets/javascripts/blazer/ace/ace.js +11 -0
  17. data/app/assets/javascripts/blazer/ace/ext-language_tools.js +5 -0
  18. data/app/assets/javascripts/blazer/ace/mode-sql.js +1 -0
  19. data/app/assets/javascripts/blazer/ace/snippets/sql.js +1 -0
  20. data/app/assets/javascripts/blazer/ace/snippets/text.js +1 -0
  21. data/app/assets/javascripts/blazer/ace/theme-twilight.js +1 -0
  22. data/app/assets/javascripts/blazer/application.js +79 -0
  23. data/app/assets/javascripts/blazer/bootstrap.js +2366 -0
  24. data/app/assets/javascripts/blazer/chartkick.js +1693 -0
  25. data/app/assets/javascripts/blazer/daterangepicker.js +1505 -0
  26. data/app/assets/javascripts/blazer/fuzzysearch.js +24 -0
  27. data/app/assets/javascripts/blazer/highlight.pack.js +1 -0
  28. data/app/assets/javascripts/blazer/jquery.js +10308 -0
  29. data/app/assets/javascripts/blazer/jquery.stickytableheaders.js +263 -0
  30. data/app/assets/javascripts/blazer/jquery_ujs.js +469 -0
  31. data/app/assets/javascripts/blazer/moment-timezone.js +1007 -0
  32. data/app/assets/javascripts/blazer/moment.js +3043 -0
  33. data/app/assets/javascripts/blazer/queries.js +110 -0
  34. data/app/assets/javascripts/blazer/routes.js +23 -0
  35. data/app/assets/javascripts/blazer/selectize.js +3667 -0
  36. data/app/assets/javascripts/blazer/stupidtable.js +114 -0
  37. data/app/assets/javascripts/blazer/vue.js +7515 -0
  38. data/app/assets/stylesheets/blazer/application.css +198 -0
  39. data/app/assets/stylesheets/blazer/bootstrap.css.erb +6202 -0
  40. data/app/assets/stylesheets/blazer/daterangepicker-bs3.css +375 -0
  41. data/app/assets/stylesheets/blazer/github.css +125 -0
  42. data/app/assets/stylesheets/blazer/selectize.default.css +387 -0
  43. data/app/controllers/blazer/base_controller.rb +103 -0
  44. data/app/controllers/blazer/checks_controller.rb +56 -0
  45. data/app/controllers/blazer/dashboards_controller.rb +105 -0
  46. data/app/controllers/blazer/queries_controller.rb +325 -0
  47. data/app/helpers/blazer/base_helper.rb +57 -0
  48. data/app/mailers/blazer/check_mailer.rb +27 -0
  49. data/app/models/blazer/audit.rb +6 -0
  50. data/app/models/blazer/check.rb +95 -0
  51. data/app/models/blazer/connection.rb +5 -0
  52. data/app/models/blazer/dashboard.rb +13 -0
  53. data/app/models/blazer/dashboard_query.rb +9 -0
  54. data/app/models/blazer/query.rb +31 -0
  55. data/app/models/blazer/record.rb +5 -0
  56. data/app/views/blazer/_nav.html.erb +16 -0
  57. data/app/views/blazer/_variables.html.erb +102 -0
  58. data/app/views/blazer/check_mailer/failing_checks.html.erb +6 -0
  59. data/app/views/blazer/check_mailer/state_change.html.erb +47 -0
  60. data/app/views/blazer/checks/_form.html.erb +71 -0
  61. data/app/views/blazer/checks/edit.html.erb +1 -0
  62. data/app/views/blazer/checks/index.html.erb +40 -0
  63. data/app/views/blazer/checks/new.html.erb +1 -0
  64. data/app/views/blazer/dashboards/_form.html.erb +76 -0
  65. data/app/views/blazer/dashboards/edit.html.erb +1 -0
  66. data/app/views/blazer/dashboards/new.html.erb +1 -0
  67. data/app/views/blazer/dashboards/show.html.erb +47 -0
  68. data/app/views/blazer/queries/_form.html.erb +240 -0
  69. data/app/views/blazer/queries/edit.html.erb +2 -0
  70. data/app/views/blazer/queries/home.html.erb +152 -0
  71. data/app/views/blazer/queries/new.html.erb +2 -0
  72. data/app/views/blazer/queries/run.html.erb +163 -0
  73. data/app/views/blazer/queries/schema.html.erb +18 -0
  74. data/app/views/blazer/queries/show.html.erb +73 -0
  75. data/app/views/layouts/blazer/application.html.erb +24 -0
  76. data/blazer.gemspec +26 -0
  77. data/config/routes.rb +16 -0
  78. data/lib/blazer.rb +185 -0
  79. data/lib/blazer/adapters/athena_adapter.rb +128 -0
  80. data/lib/blazer/adapters/base_adapter.rb +53 -0
  81. data/lib/blazer/adapters/bigquery_adapter.rb +67 -0
  82. data/lib/blazer/adapters/drill_adapter.rb +28 -0
  83. data/lib/blazer/adapters/elasticsearch_adapter.rb +49 -0
  84. data/lib/blazer/adapters/mongodb_adapter.rb +39 -0
  85. data/lib/blazer/adapters/presto_adapter.rb +45 -0
  86. data/lib/blazer/adapters/sql_adapter.rb +182 -0
  87. data/lib/blazer/data_source.rb +193 -0
  88. data/lib/blazer/detect_anomalies.R +19 -0
  89. data/lib/blazer/engine.rb +47 -0
  90. data/lib/blazer/result.rb +170 -0
  91. data/lib/blazer/run_statement.rb +40 -0
  92. data/lib/blazer/run_statement_job.rb +21 -0
  93. data/lib/blazer/version.rb +3 -0
  94. data/lib/generators/blazer/install_generator.rb +39 -0
  95. data/lib/generators/blazer/templates/config.yml +62 -0
  96. data/lib/generators/blazer/templates/install.rb +45 -0
  97. data/lib/tasks/blazer.rake +10 -0
  98. metadata +211 -0
@@ -0,0 +1,193 @@
1
+ require "digest/md5"
2
+
3
+ module Blazer
4
+ class DataSource
5
+ extend Forwardable
6
+
7
+ attr_reader :id, :settings, :adapter, :adapter_instance
8
+
9
+ def_delegators :adapter_instance, :schema, :tables, :preview_statement, :reconnect, :cost, :explain, :cancel
10
+
11
+ def initialize(id, settings)
12
+ @id = id
13
+ @settings = settings
14
+
15
+ unless settings["url"] || Rails.env.development? || ["bigquery", "athena"].include?(settings["adapter"])
16
+ raise Blazer::Error, "Empty url for data source: #{id}"
17
+ end
18
+
19
+ @adapter_instance =
20
+ if Blazer.adapters[adapter]
21
+ Blazer.adapters[adapter].new(self)
22
+ else
23
+ raise Blazer::Error, "Unknown adapter"
24
+ end
25
+ end
26
+
27
+ def adapter
28
+ settings["adapter"] || detect_adapter
29
+ end
30
+
31
+ def name
32
+ settings["name"] || @id
33
+ end
34
+
35
+ def linked_columns
36
+ settings["linked_columns"] || {}
37
+ end
38
+
39
+ def smart_columns
40
+ settings["smart_columns"] || {}
41
+ end
42
+
43
+ def smart_variables
44
+ settings["smart_variables"] || {}
45
+ end
46
+
47
+ def variable_defaults
48
+ settings["variable_defaults"] || {}
49
+ end
50
+
51
+ def timeout
52
+ settings["timeout"]
53
+ end
54
+
55
+ def cache
56
+ @cache ||= begin
57
+ if settings["cache"].is_a?(Hash)
58
+ settings["cache"]
59
+ elsif settings["cache"]
60
+ {
61
+ "mode" => "all",
62
+ "expires_in" => settings["cache"]
63
+ }
64
+ else
65
+ {
66
+ "mode" => "off"
67
+ }
68
+ end
69
+ end
70
+ end
71
+
72
+ def cache_mode
73
+ cache["mode"]
74
+ end
75
+
76
+ def cache_expires_in
77
+ (cache["expires_in"] || 60).to_f
78
+ end
79
+
80
+ def cache_slow_threshold
81
+ (cache["slow_threshold"] || 15).to_f
82
+ end
83
+
84
+ def local_time_suffix
85
+ @local_time_suffix ||= Array(settings["local_time_suffix"])
86
+ end
87
+
88
+ def read_cache(cache_key)
89
+ value = Blazer.cache.read(cache_key)
90
+ if value
91
+ Blazer::Result.new(self, *Marshal.load(value), nil)
92
+ end
93
+ end
94
+
95
+ def run_results(run_id)
96
+ read_cache(run_cache_key(run_id))
97
+ end
98
+
99
+ def delete_results(run_id)
100
+ Blazer.cache.delete(run_cache_key(run_id))
101
+ end
102
+
103
+ def run_statement(statement, options = {})
104
+ run_id = options[:run_id]
105
+ async = options[:async]
106
+ result = nil
107
+ if cache_mode != "off"
108
+ if options[:refresh_cache]
109
+ clear_cache(statement) # for checks
110
+ else
111
+ result = read_cache(statement_cache_key(statement))
112
+ end
113
+ end
114
+
115
+ unless result
116
+ comment = "blazer"
117
+ if options[:user].respond_to?(:id)
118
+ comment << ",user_id:#{options[:user].id}"
119
+ end
120
+ if options[:user].respond_to?(Blazer.user_name)
121
+ # only include letters, numbers, and spaces to prevent injection
122
+ comment << ",user_name:#{options[:user].send(Blazer.user_name).to_s.gsub(/[^a-zA-Z0-9 ]/, "")}"
123
+ end
124
+ if options[:query].respond_to?(:id)
125
+ comment << ",query_id:#{options[:query].id}"
126
+ end
127
+ if options[:check]
128
+ comment << ",check_id:#{options[:check].id},check_emails:#{options[:check].emails}"
129
+ end
130
+ if options[:run_id]
131
+ comment << ",run_id:#{options[:run_id]}"
132
+ end
133
+ result = run_statement_helper(statement, comment, async ? options[:run_id] : nil)
134
+ end
135
+
136
+ result
137
+ end
138
+
139
+ def clear_cache(statement)
140
+ Blazer.cache.delete(statement_cache_key(statement))
141
+ end
142
+
143
+ def cache_key(key)
144
+ (["blazer", "v4"] + key).join("/")
145
+ end
146
+
147
+ def statement_cache_key(statement)
148
+ cache_key(["statement", id, Digest::MD5.hexdigest(statement.to_s.gsub("\r\n", "\n"))])
149
+ end
150
+
151
+ def run_cache_key(run_id)
152
+ cache_key(["run", run_id])
153
+ end
154
+
155
+ protected
156
+
157
+ def run_statement_helper(statement, comment, run_id)
158
+ start_time = Time.now
159
+ columns, rows, error = @adapter_instance.run_statement(statement, comment)
160
+ duration = Time.now - start_time
161
+
162
+ cache_data = nil
163
+ cache = !error && (cache_mode == "all" || (cache_mode == "slow" && duration >= cache_slow_threshold))
164
+ if cache || run_id
165
+ cache_data = Marshal.dump([columns, rows, error, cache ? Time.now : nil]) rescue nil
166
+ end
167
+
168
+ if cache && cache_data && @adapter_instance.cachable?(statement)
169
+ Blazer.cache.write(statement_cache_key(statement), cache_data, expires_in: cache_expires_in.to_f * 60)
170
+ end
171
+
172
+ if run_id
173
+ unless cache_data
174
+ error = "Error storing the results of this query :("
175
+ cache_data = Marshal.dump([[], [], error, nil])
176
+ end
177
+ Blazer.cache.write(run_cache_key(run_id), cache_data, expires_in: 30.seconds)
178
+ end
179
+
180
+ Blazer::Result.new(self, columns, rows, error, nil, cache && !cache_data.nil?)
181
+ end
182
+
183
+ def detect_adapter
184
+ schema = settings["url"].to_s.split("://").first
185
+ case schema
186
+ when "mongodb", "presto"
187
+ schema
188
+ else
189
+ "sql"
190
+ end
191
+ end
192
+ end
193
+ end
@@ -0,0 +1,19 @@
1
+ tryCatch({
2
+ library(AnomalyDetection)
3
+
4
+ args <- commandArgs(trailingOnly = TRUE)
5
+
6
+ con <- textConnection(args[2])
7
+ data <- read.csv(con, stringsAsFactors = FALSE)
8
+ data$timestamp <- as.POSIXct(data$timestamp)
9
+
10
+ if (identical(args[1], "ts")) {
11
+ res <- AnomalyDetectionTs(data, direction = "both", alpha = 0.05, max_anoms = 0.2)
12
+ } else {
13
+ res <- AnomalyDetectionVec(data$count, direction = "both", alpha = 0.05, max_anoms = 0.2, period = length(data$count) / 2 - 1)
14
+ }
15
+
16
+ write.csv(res$anoms)
17
+ }, error = function (e) {
18
+ write.csv(geterrmessage())
19
+ })
@@ -0,0 +1,47 @@
1
+ module Blazer
2
+ class Engine < ::Rails::Engine
3
+ isolate_namespace Blazer
4
+
5
+ initializer "blazer" do |app|
6
+ # use a proc instead of a string
7
+ app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/application\.(js|css)\z/ }
8
+ app.config.assets.precompile << proc { |path| path =~ /\Ablazer\/.+\.(eot|svg|ttf|woff)\z/ }
9
+
10
+ Blazer.time_zone ||= Blazer.settings["time_zone"] || Time.zone
11
+ Blazer.audit = Blazer.settings.key?("audit") ? Blazer.settings["audit"] : true
12
+ Blazer.user_name = Blazer.settings["user_name"] if Blazer.settings["user_name"]
13
+ Blazer.from_email = Blazer.settings["from_email"] if Blazer.settings["from_email"]
14
+ Blazer.before_action = Blazer.settings["before_action"] if Blazer.settings["before_action"]
15
+
16
+ Blazer.user_class ||= Blazer.settings.key?("user_class") ? Blazer.settings["user_class"] : (User rescue nil)
17
+ Blazer.user_method = Blazer.settings["user_method"]
18
+ if Blazer.user_class
19
+ Blazer.user_method ||= "current_#{Blazer.user_class.to_s.downcase.singularize}"
20
+ end
21
+
22
+ Blazer.check_schedules = Blazer.settings["check_schedules"] if Blazer.settings.key?("check_schedules")
23
+ if Blazer.settings.key?("mapbox_access_token")
24
+ Blazer.mapbox_access_token = Blazer.settings["mapbox_access_token"]
25
+ elsif ENV["MAPBOX_ACCESS_TOKEN"].present?
26
+ Blazer.mapbox_access_token = ENV["MAPBOX_ACCESS_TOKEN"]
27
+ end
28
+
29
+ if Blazer.user_class
30
+ options = Blazer::BELONGS_TO_OPTIONAL.merge(class_name: Blazer.user_class.to_s)
31
+ Blazer::Query.belongs_to :creator, options
32
+ Blazer::Dashboard.belongs_to :creator, options
33
+ Blazer::Check.belongs_to :creator, options
34
+ end
35
+
36
+ Blazer.cache ||= Rails.cache
37
+
38
+ Blazer.anomaly_checks = Blazer.settings["anomaly_checks"] || false
39
+ Blazer.async = Blazer.settings["async"] || false
40
+ if Blazer.async
41
+ require "blazer/run_statement_job"
42
+ end
43
+
44
+ Blazer.images = Blazer.settings["images"] || false
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,170 @@
1
+ module Blazer
2
+ class Result
3
+ attr_reader :data_source, :columns, :rows, :error, :cached_at, :just_cached
4
+
5
+ def initialize(data_source, columns, rows, error, cached_at, just_cached)
6
+ @data_source = data_source
7
+ @columns = columns
8
+ @rows = rows
9
+ @error = error
10
+ @cached_at = cached_at
11
+ @just_cached = just_cached
12
+ end
13
+
14
+ def timed_out?
15
+ error == Blazer::TIMEOUT_MESSAGE
16
+ end
17
+
18
+ def cached?
19
+ cached_at.present?
20
+ end
21
+
22
+ def boom
23
+ @boom ||= begin
24
+ boom = {}
25
+ columns.each_with_index do |key, i|
26
+ smart_columns_data_source =
27
+ ([data_source] + Array(data_source.settings["inherit_smart_settings"]).map { |ds| Blazer.data_sources[ds] }).find { |ds| ds.smart_columns[key] }
28
+
29
+ if smart_columns_data_source
30
+ query = smart_columns_data_source.smart_columns[key]
31
+ res =
32
+ if query.is_a?(Hash)
33
+ query
34
+ else
35
+ values = rows.map { |r| r[i] }.compact.uniq
36
+ result = smart_columns_data_source.run_statement(ActiveRecord::Base.send(:sanitize_sql_array, [query.sub("{value}", "(?)"), values]))
37
+ result.rows
38
+ end
39
+
40
+ boom[key] = Hash[res.map { |k, v| [k.nil? ? k : k.to_s, v] }]
41
+ end
42
+ end
43
+ boom
44
+ end
45
+ end
46
+
47
+ def column_types
48
+ @column_types ||= begin
49
+ columns.each_with_index.map do |k, i|
50
+ v = (rows.find { |r| r[i] } || {})[i]
51
+ if boom[k]
52
+ "string"
53
+ elsif v.is_a?(Numeric)
54
+ "numeric"
55
+ elsif v.is_a?(Time) || v.is_a?(Date)
56
+ "time"
57
+ elsif v.nil?
58
+ nil
59
+ else
60
+ "string"
61
+ end
62
+ end
63
+ end
64
+ end
65
+
66
+ def chart_type
67
+ @chart_type ||= begin
68
+ if column_types.compact.size >= 2 && column_types.compact == ["time"] + (column_types.compact.size - 1).times.map { "numeric" }
69
+ "line"
70
+ elsif column_types == ["time", "string", "numeric"]
71
+ "line2"
72
+ elsif column_types.compact.size >= 2 && column_types == ["string"] + (column_types.compact.size - 1).times.map { "numeric" }
73
+ "bar"
74
+ elsif column_types == ["string", "string", "numeric"]
75
+ "bar2"
76
+ elsif column_types == ["numeric", "numeric"]
77
+ "scatter"
78
+ end
79
+ end
80
+ end
81
+
82
+ def detect_anomaly
83
+ anomaly = nil
84
+ message = nil
85
+
86
+ if rows.empty?
87
+ message = "No data"
88
+ else
89
+ if chart_type == "line" || chart_type == "line2"
90
+ series = []
91
+
92
+ if chart_type == "line"
93
+ columns[1..-1].each_with_index.each do |k, i|
94
+ series << {name: k, data: rows.map{ |r| [r[0], r[i + 1]] }}
95
+ end
96
+ else
97
+ rows.group_by { |r| v = r[1]; (boom[columns[1]] || {})[v.to_s] || v }.each_with_index.map do |(name, v), i|
98
+ series << {name: name, data: v.map { |v2| [v2[0], v2[2]] }}
99
+ end
100
+ end
101
+
102
+ current_series = nil
103
+ begin
104
+ anomalies = []
105
+ series.each do |s|
106
+ current_series = s[:name]
107
+ anomalies << s[:name] if anomaly?(s[:data])
108
+ end
109
+ anomaly = anomalies.any?
110
+ if anomaly
111
+ if anomalies.size == 1
112
+ message = "Anomaly detected in #{anomalies.first}"
113
+ else
114
+ message = "Anomalies detected in #{anomalies.to_sentence}"
115
+ end
116
+ else
117
+ message = "No anomalies detected"
118
+ end
119
+ rescue => e
120
+ message = "#{current_series}: #{e.message}"
121
+ raise e if Rails.env.development?
122
+ end
123
+ else
124
+ message = "Bad format"
125
+ end
126
+ end
127
+
128
+ [anomaly, message]
129
+ end
130
+
131
+ def anomaly?(series)
132
+ series = series.reject { |v| v[0].nil? }.sort_by { |v| v[0] }
133
+
134
+ csv_str =
135
+ CSV.generate do |csv|
136
+ csv << ["timestamp", "count"]
137
+ series.each do |row|
138
+ csv << row
139
+ end
140
+ end
141
+
142
+ r_script = %x[which Rscript].chomp
143
+ type = series.any? && series.last.first.to_time - series.first.first.to_time >= 2.weeks ? "ts" : "vec"
144
+ args = [type, csv_str]
145
+ raise "R not found" if r_script.empty?
146
+ command = "#{r_script} --vanilla #{File.expand_path("../detect_anomalies.R", __FILE__)} #{args.map { |a| Shellwords.escape(a) }.join(" ")}"
147
+ output = %x[#{command}]
148
+ if output.empty?
149
+ raise "Unknown R error"
150
+ end
151
+
152
+ rows = CSV.parse(output, headers: true)
153
+ error = rows.first && rows.first["x"]
154
+ raise error if error
155
+
156
+ timestamps = []
157
+ if type == "ts"
158
+ rows.each do |row|
159
+ timestamps << Time.parse(row["timestamp"])
160
+ end
161
+ timestamps.include?(series.last[0].to_time)
162
+ else
163
+ rows.each do |row|
164
+ timestamps << row["index"].to_i
165
+ end
166
+ timestamps.include?(series.length)
167
+ end
168
+ end
169
+ end
170
+ end
@@ -0,0 +1,40 @@
1
+ module Blazer
2
+ class RunStatement
3
+ def perform(data_source, statement, options = {})
4
+ query = options[:query]
5
+ Blazer.transform_statement.call(data_source, statement) if Blazer.transform_statement
6
+
7
+ # audit
8
+ if Blazer.audit
9
+ audit = Blazer::Audit.new(statement: statement)
10
+ audit.query = query
11
+ audit.data_source = data_source.id
12
+ audit.user = options[:user]
13
+ audit.save!
14
+ end
15
+
16
+ start_time = Time.now
17
+ result = data_source.run_statement(statement, options)
18
+ duration = Time.now - start_time
19
+
20
+ if Blazer.audit
21
+ audit.duration = duration if audit.respond_to?(:duration=)
22
+ audit.error = result.error if audit.respond_to?(:error=)
23
+ audit.timed_out = result.timed_out? if audit.respond_to?(:timed_out=)
24
+ audit.cached = result.cached? if audit.respond_to?(:cached=)
25
+ if !result.cached? && duration >= 10
26
+ audit.cost = data_source.cost(statement) if audit.respond_to?(:cost=)
27
+ end
28
+ audit.save! if audit.changed?
29
+ end
30
+
31
+ if query && !result.timed_out? && !query.variables.any?
32
+ query.checks.each do |check|
33
+ check.update_state(result)
34
+ end
35
+ end
36
+
37
+ result
38
+ end
39
+ end
40
+ end