finery 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +426 -0
- data/CONTRIBUTING.md +49 -0
- data/LICENSE.txt +25 -0
- data/README.md +1144 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.eot +0 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.svg +288 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.ttf +0 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff +0 -0
- data/app/assets/fonts/blazer/glyphicons-halflings-regular.woff2 +0 -0
- data/app/assets/images/blazer/favicon.png +0 -0
- data/app/assets/javascripts/blazer/Sortable.js +3709 -0
- data/app/assets/javascripts/blazer/ace/ace.js +19630 -0
- data/app/assets/javascripts/blazer/ace/ext-language_tools.js +1981 -0
- data/app/assets/javascripts/blazer/ace/mode-sql.js +215 -0
- data/app/assets/javascripts/blazer/ace/snippets/sql.js +16 -0
- data/app/assets/javascripts/blazer/ace/snippets/text.js +9 -0
- data/app/assets/javascripts/blazer/ace/theme-twilight.js +18 -0
- data/app/assets/javascripts/blazer/ace.js +6 -0
- data/app/assets/javascripts/blazer/application.js +87 -0
- data/app/assets/javascripts/blazer/bootstrap.js +2580 -0
- data/app/assets/javascripts/blazer/chart.umd.js +13 -0
- data/app/assets/javascripts/blazer/chartjs-adapter-date-fns.bundle.js +6322 -0
- data/app/assets/javascripts/blazer/chartjs-plugin-annotation.min.js +7 -0
- data/app/assets/javascripts/blazer/chartkick.js +2570 -0
- data/app/assets/javascripts/blazer/daterangepicker.js +1578 -0
- data/app/assets/javascripts/blazer/fuzzysearch.js +24 -0
- data/app/assets/javascripts/blazer/highlight.min.js +466 -0
- data/app/assets/javascripts/blazer/jquery.js +10872 -0
- data/app/assets/javascripts/blazer/jquery.stickytableheaders.js +325 -0
- data/app/assets/javascripts/blazer/mapkick.bundle.js +1029 -0
- data/app/assets/javascripts/blazer/moment-timezone-with-data.js +1548 -0
- data/app/assets/javascripts/blazer/moment.js +5685 -0
- data/app/assets/javascripts/blazer/queries.js +130 -0
- data/app/assets/javascripts/blazer/rails-ujs.js +746 -0
- data/app/assets/javascripts/blazer/routes.js +26 -0
- data/app/assets/javascripts/blazer/selectize.js +3891 -0
- data/app/assets/javascripts/blazer/stupidtable-custom-settings.js +13 -0
- data/app/assets/javascripts/blazer/stupidtable.js +281 -0
- data/app/assets/javascripts/blazer/vue.global.prod.js +1 -0
- data/app/assets/stylesheets/blazer/application.css +243 -0
- data/app/assets/stylesheets/blazer/bootstrap-propshaft.css +10 -0
- data/app/assets/stylesheets/blazer/bootstrap-sprockets.css.erb +10 -0
- data/app/assets/stylesheets/blazer/bootstrap.css +6828 -0
- data/app/assets/stylesheets/blazer/daterangepicker.css +410 -0
- data/app/assets/stylesheets/blazer/github.css +125 -0
- data/app/assets/stylesheets/blazer/selectize.css +403 -0
- data/app/controllers/blazer/base_controller.rb +133 -0
- data/app/controllers/blazer/checks_controller.rb +56 -0
- data/app/controllers/blazer/dashboards_controller.rb +99 -0
- data/app/controllers/blazer/queries_controller.rb +468 -0
- data/app/controllers/blazer/uploads_controller.rb +147 -0
- data/app/helpers/blazer/base_helper.rb +83 -0
- data/app/models/blazer/audit.rb +6 -0
- data/app/models/blazer/check.rb +104 -0
- data/app/models/blazer/connection.rb +5 -0
- data/app/models/blazer/dashboard.rb +17 -0
- data/app/models/blazer/dashboard_query.rb +9 -0
- data/app/models/blazer/query.rb +42 -0
- data/app/models/blazer/record.rb +5 -0
- data/app/models/blazer/upload.rb +11 -0
- data/app/models/blazer/uploads_connection.rb +7 -0
- data/app/views/blazer/_nav.html.erb +18 -0
- data/app/views/blazer/_variables.html.erb +127 -0
- data/app/views/blazer/check_mailer/failing_checks.html.erb +7 -0
- data/app/views/blazer/check_mailer/state_change.html.erb +48 -0
- data/app/views/blazer/checks/_form.html.erb +79 -0
- data/app/views/blazer/checks/edit.html.erb +3 -0
- data/app/views/blazer/checks/index.html.erb +72 -0
- data/app/views/blazer/checks/new.html.erb +3 -0
- data/app/views/blazer/dashboards/_form.html.erb +82 -0
- data/app/views/blazer/dashboards/edit.html.erb +3 -0
- data/app/views/blazer/dashboards/new.html.erb +3 -0
- data/app/views/blazer/dashboards/show.html.erb +53 -0
- data/app/views/blazer/queries/_caching.html.erb +16 -0
- data/app/views/blazer/queries/_cohorts.html.erb +48 -0
- data/app/views/blazer/queries/_form.html.erb +255 -0
- data/app/views/blazer/queries/docs.html.erb +147 -0
- data/app/views/blazer/queries/edit.html.erb +2 -0
- data/app/views/blazer/queries/home.html.erb +169 -0
- data/app/views/blazer/queries/new.html.erb +2 -0
- data/app/views/blazer/queries/run.html.erb +188 -0
- data/app/views/blazer/queries/schema.html.erb +55 -0
- data/app/views/blazer/queries/show.html.erb +72 -0
- data/app/views/blazer/uploads/_form.html.erb +27 -0
- data/app/views/blazer/uploads/edit.html.erb +3 -0
- data/app/views/blazer/uploads/index.html.erb +55 -0
- data/app/views/blazer/uploads/new.html.erb +3 -0
- data/app/views/layouts/blazer/application.html.erb +25 -0
- data/config/routes.rb +25 -0
- data/lib/blazer/adapters/athena_adapter.rb +182 -0
- data/lib/blazer/adapters/base_adapter.rb +76 -0
- data/lib/blazer/adapters/bigquery_adapter.rb +79 -0
- data/lib/blazer/adapters/cassandra_adapter.rb +70 -0
- data/lib/blazer/adapters/clickhouse_adapter.rb +84 -0
- data/lib/blazer/adapters/drill_adapter.rb +38 -0
- data/lib/blazer/adapters/druid_adapter.rb +102 -0
- data/lib/blazer/adapters/elasticsearch_adapter.rb +61 -0
- data/lib/blazer/adapters/hive_adapter.rb +55 -0
- data/lib/blazer/adapters/ignite_adapter.rb +64 -0
- data/lib/blazer/adapters/influxdb_adapter.rb +57 -0
- data/lib/blazer/adapters/neo4j_adapter.rb +62 -0
- data/lib/blazer/adapters/opensearch_adapter.rb +52 -0
- data/lib/blazer/adapters/presto_adapter.rb +54 -0
- data/lib/blazer/adapters/salesforce_adapter.rb +50 -0
- data/lib/blazer/adapters/snowflake_adapter.rb +82 -0
- data/lib/blazer/adapters/soda_adapter.rb +105 -0
- data/lib/blazer/adapters/spark_adapter.rb +14 -0
- data/lib/blazer/adapters/sql_adapter.rb +324 -0
- data/lib/blazer/adapters.rb +18 -0
- data/lib/blazer/annotations.rb +47 -0
- data/lib/blazer/anomaly_detectors.rb +22 -0
- data/lib/blazer/check_mailer.rb +27 -0
- data/lib/blazer/data_source.rb +270 -0
- data/lib/blazer/engine.rb +42 -0
- data/lib/blazer/forecasters.rb +7 -0
- data/lib/blazer/result.rb +178 -0
- data/lib/blazer/result_cache.rb +71 -0
- data/lib/blazer/run_statement.rb +44 -0
- data/lib/blazer/run_statement_job.rb +20 -0
- data/lib/blazer/slack_notifier.rb +94 -0
- data/lib/blazer/statement.rb +77 -0
- data/lib/blazer/version.rb +3 -0
- data/lib/blazer.rb +286 -0
- data/lib/finery.rb +3 -0
- data/lib/generators/blazer/install_generator.rb +22 -0
- data/lib/generators/blazer/templates/config.yml.tt +83 -0
- data/lib/generators/blazer/templates/install.rb.tt +47 -0
- data/lib/generators/blazer/templates/uploads.rb.tt +10 -0
- data/lib/generators/blazer/uploads_generator.rb +18 -0
- data/lib/tasks/blazer.rake +20 -0
- data/lib/tasks/finery.rake +20 -0
- data/licenses/LICENSE-ace.txt +24 -0
- data/licenses/LICENSE-bootstrap.txt +21 -0
- data/licenses/LICENSE-chart.js.txt +9 -0
- data/licenses/LICENSE-chartjs-adapter-date-fns.txt +9 -0
- data/licenses/LICENSE-chartkick.js.txt +22 -0
- data/licenses/LICENSE-date-fns.txt +21 -0
- data/licenses/LICENSE-daterangepicker.txt +21 -0
- data/licenses/LICENSE-fuzzysearch.txt +20 -0
- data/licenses/LICENSE-highlight.js.txt +29 -0
- data/licenses/LICENSE-jquery.txt +20 -0
- data/licenses/LICENSE-kurkle-color.txt +9 -0
- data/licenses/LICENSE-mapkick-bundle.txt +1029 -0
- data/licenses/LICENSE-moment-timezone.txt +20 -0
- data/licenses/LICENSE-moment.txt +22 -0
- data/licenses/LICENSE-rails-ujs.txt +20 -0
- data/licenses/LICENSE-selectize.txt +202 -0
- data/licenses/LICENSE-sortable.txt +21 -0
- data/licenses/LICENSE-stickytableheaders.txt +20 -0
- data/licenses/LICENSE-stupidtable.txt +19 -0
- data/licenses/LICENSE-vue.txt +21 -0
- metadata +250 -0
@@ -0,0 +1,62 @@
|
|
1
|
+
module Blazer
|
2
|
+
module Adapters
|
3
|
+
class Neo4jAdapter < BaseAdapter
|
4
|
+
def run_statement(statement, comment, bind_params)
|
5
|
+
columns = []
|
6
|
+
rows = []
|
7
|
+
error = nil
|
8
|
+
|
9
|
+
begin
|
10
|
+
result = session.query("#{statement} /*#{comment}*/", bind_params)
|
11
|
+
columns = result.columns.map(&:to_s)
|
12
|
+
rows = []
|
13
|
+
result.each do |row|
|
14
|
+
rows << columns.map do |c|
|
15
|
+
v = row.send(c)
|
16
|
+
v = v.properties if v.respond_to?(:properties)
|
17
|
+
v
|
18
|
+
end
|
19
|
+
end
|
20
|
+
rescue => e
|
21
|
+
error = e.message
|
22
|
+
error = Blazer::VARIABLE_MESSAGE if error.include?("Invalid input '$'")
|
23
|
+
end
|
24
|
+
|
25
|
+
[columns, rows, error]
|
26
|
+
end
|
27
|
+
|
28
|
+
def tables
|
29
|
+
result = session.query("CALL db.labels()")
|
30
|
+
result.rows.map(&:first)
|
31
|
+
end
|
32
|
+
|
33
|
+
def preview_statement
|
34
|
+
"MATCH (n:{table}) RETURN n LIMIT 10"
|
35
|
+
end
|
36
|
+
|
37
|
+
# https://neo4j.com/docs/cypher-manual/current/syntax/expressions/#cypher-expressions-string-literals
|
38
|
+
def quoting
|
39
|
+
:backslash_escape
|
40
|
+
end
|
41
|
+
|
42
|
+
def parameter_binding
|
43
|
+
proc do |statement, variables|
|
44
|
+
variables.each_key do |k|
|
45
|
+
statement = statement.gsub("{#{k}}") { "$#{k} " }
|
46
|
+
end
|
47
|
+
[statement, variables]
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
protected
|
52
|
+
|
53
|
+
def session
|
54
|
+
@session ||= begin
|
55
|
+
require "neo4j/core/cypher_session/adaptors/http"
|
56
|
+
http_adaptor = Neo4j::Core::CypherSession::Adaptors::HTTP.new(settings["url"])
|
57
|
+
Neo4j::Core::CypherSession.new(http_adaptor)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module Blazer
|
2
|
+
module Adapters
|
3
|
+
class OpensearchAdapter < BaseAdapter
|
4
|
+
def run_statement(statement, comment)
|
5
|
+
columns = []
|
6
|
+
rows = []
|
7
|
+
error = nil
|
8
|
+
|
9
|
+
begin
|
10
|
+
response = client.transport.perform_request("POST", "_plugins/_sql", {}, {query: "#{statement} /*#{comment}*/"}).body
|
11
|
+
columns = response["schema"].map { |v| v["name"] }
|
12
|
+
# TODO typecast more types
|
13
|
+
# https://github.com/opensearch-project/sql/blob/main/docs/user/general/datatypes.rst
|
14
|
+
date_indexes = response["schema"].each_index.select { |i| response["schema"][i]["type"] == "timestamp" }
|
15
|
+
if columns.any?
|
16
|
+
rows = response["datarows"]
|
17
|
+
utc = ActiveSupport::TimeZone["Etc/UTC"]
|
18
|
+
date_indexes.each do |i|
|
19
|
+
rows.each do |row|
|
20
|
+
row[i] &&= utc.parse(row[i])
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
rescue => e
|
25
|
+
error = e.message
|
26
|
+
end
|
27
|
+
|
28
|
+
[columns, rows, error]
|
29
|
+
end
|
30
|
+
|
31
|
+
def tables
|
32
|
+
indices = client.cat.indices(format: "json").map { |v| v["index"] }
|
33
|
+
aliases = client.cat.aliases(format: "json").map { |v| v["alias"] }
|
34
|
+
(indices + aliases).uniq.sort
|
35
|
+
end
|
36
|
+
|
37
|
+
def preview_statement
|
38
|
+
"SELECT * FROM `{table}` LIMIT 10"
|
39
|
+
end
|
40
|
+
|
41
|
+
def quoting
|
42
|
+
# unknown
|
43
|
+
end
|
44
|
+
|
45
|
+
protected
|
46
|
+
|
47
|
+
def client
|
48
|
+
@client ||= OpenSearch::Client.new(url: settings["url"])
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Blazer
|
2
|
+
module Adapters
|
3
|
+
class PrestoAdapter < BaseAdapter
|
4
|
+
def run_statement(statement, comment)
|
5
|
+
columns = []
|
6
|
+
rows = []
|
7
|
+
error = nil
|
8
|
+
|
9
|
+
begin
|
10
|
+
columns, rows = client.run("#{statement} /*#{comment}*/")
|
11
|
+
columns = columns.map(&:name)
|
12
|
+
rescue => e
|
13
|
+
error = e.message
|
14
|
+
end
|
15
|
+
|
16
|
+
[columns, rows, error]
|
17
|
+
end
|
18
|
+
|
19
|
+
def tables
|
20
|
+
_, rows = client.run("SHOW TABLES")
|
21
|
+
rows.map(&:first)
|
22
|
+
end
|
23
|
+
|
24
|
+
def preview_statement
|
25
|
+
"SELECT * FROM {table} LIMIT 10"
|
26
|
+
end
|
27
|
+
|
28
|
+
def quoting
|
29
|
+
:single_quote_escape
|
30
|
+
end
|
31
|
+
|
32
|
+
# TODO support prepared statements - https://prestodb.io/docs/current/sql/prepare.html
|
33
|
+
# feature request for variables - https://github.com/prestodb/presto/issues/5918
|
34
|
+
def parameter_binding
|
35
|
+
end
|
36
|
+
|
37
|
+
protected
|
38
|
+
|
39
|
+
def client
|
40
|
+
@client ||= begin
|
41
|
+
uri = URI.parse(settings["url"])
|
42
|
+
query = uri.query ? CGI::parse(uri.query) : {}
|
43
|
+
Presto::Client.new(
|
44
|
+
server: "#{uri.host}:#{uri.port}",
|
45
|
+
catalog: uri.path.to_s.sub(/\A\//, ""),
|
46
|
+
schema: query["schema"] || "public",
|
47
|
+
user: uri.user,
|
48
|
+
http_debug: false
|
49
|
+
)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module Blazer
|
2
|
+
module Adapters
|
3
|
+
class SalesforceAdapter < BaseAdapter
|
4
|
+
def run_statement(statement, comment)
|
5
|
+
columns = []
|
6
|
+
rows = []
|
7
|
+
error = nil
|
8
|
+
|
9
|
+
# remove comments manually
|
10
|
+
statement = statement.gsub(/--.+/, "")
|
11
|
+
# only supports single line /* */ comments
|
12
|
+
# regex not perfect, but should be good enough
|
13
|
+
statement = statement.gsub(/\/\*.+\*\//, "")
|
14
|
+
|
15
|
+
# remove trailing semicolon
|
16
|
+
statement = statement.sub(/;\s*\z/, "")
|
17
|
+
|
18
|
+
begin
|
19
|
+
response = client.query(statement)
|
20
|
+
rows = response.map { |r| r.to_hash.except("attributes").values }
|
21
|
+
columns = rows.any? ? response.first.to_hash.except("attributes").keys : []
|
22
|
+
rescue => e
|
23
|
+
error = e.message
|
24
|
+
end
|
25
|
+
|
26
|
+
[columns, rows, error]
|
27
|
+
end
|
28
|
+
|
29
|
+
def tables
|
30
|
+
# cache
|
31
|
+
@tables ||= client.describe.select { |r| r.queryable }.map(&:name)
|
32
|
+
end
|
33
|
+
|
34
|
+
def preview_statement
|
35
|
+
"SELECT Id FROM {table} LIMIT 10"
|
36
|
+
end
|
37
|
+
|
38
|
+
# https://developer.salesforce.com/docs/atlas.en-us.soql_sosl.meta/soql_sosl/sforce_api_calls_soql_select_quotedstringescapes.htm
|
39
|
+
def quoting
|
40
|
+
:backslash_escape
|
41
|
+
end
|
42
|
+
|
43
|
+
protected
|
44
|
+
|
45
|
+
def client
|
46
|
+
@client ||= Restforce.new
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
module Blazer
|
2
|
+
module Adapters
|
3
|
+
class SnowflakeAdapter < SqlAdapter
|
4
|
+
def initialize(data_source)
|
5
|
+
@data_source = data_source
|
6
|
+
|
7
|
+
@@registered ||= begin
|
8
|
+
require "active_record/connection_adapters/odbc_adapter"
|
9
|
+
require "odbc_adapter/adapters/postgresql_odbc_adapter"
|
10
|
+
|
11
|
+
ODBCAdapter.register(/snowflake/, ODBCAdapter::Adapters::PostgreSQLODBCAdapter) do
|
12
|
+
# Explicitly turning off prepared statements as they are not yet working with
|
13
|
+
# snowflake + the ODBC ActiveRecord adapter
|
14
|
+
def prepared_statements
|
15
|
+
false
|
16
|
+
end
|
17
|
+
|
18
|
+
# Quoting needs to be changed for snowflake
|
19
|
+
def quote_column_name(name)
|
20
|
+
name.to_s
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
# Override dbms_type_cast to get the values encoded in UTF-8
|
26
|
+
def dbms_type_cast(columns, values)
|
27
|
+
int_column = {}
|
28
|
+
columns.each_with_index do |c, i|
|
29
|
+
int_column[i] = c.type == 3 && c.scale == 0
|
30
|
+
end
|
31
|
+
|
32
|
+
float_column = {}
|
33
|
+
columns.each_with_index do |c, i|
|
34
|
+
float_column[i] = c.type == 3 && c.scale != 0
|
35
|
+
end
|
36
|
+
|
37
|
+
values.each do |row|
|
38
|
+
row.each_index do |idx|
|
39
|
+
val = row[idx]
|
40
|
+
if val
|
41
|
+
if int_column[idx]
|
42
|
+
row[idx] = val.to_i
|
43
|
+
elsif float_column[idx]
|
44
|
+
row[idx] = val.to_f
|
45
|
+
elsif val.is_a?(String)
|
46
|
+
row[idx] = val.force_encoding('UTF-8')
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
@connection_model =
|
56
|
+
Class.new(Blazer::Connection) do
|
57
|
+
def self.name
|
58
|
+
"Blazer::Connection::SnowflakeAdapter#{object_id}"
|
59
|
+
end
|
60
|
+
if data_source.settings["conn_str"]
|
61
|
+
establish_connection(adapter: "odbc", conn_str: data_source.settings["conn_str"])
|
62
|
+
elsif data_source.settings["dsn"]
|
63
|
+
establish_connection(adapter: "odbc", dsn: data_source.settings["dsn"])
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def cancel(run_id)
|
69
|
+
# todo
|
70
|
+
end
|
71
|
+
|
72
|
+
# https://docs.snowflake.com/en/sql-reference/data-types-text.html#escape-sequences
|
73
|
+
def quoting
|
74
|
+
:backslash_escape
|
75
|
+
end
|
76
|
+
|
77
|
+
def parameter_binding
|
78
|
+
# TODO
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
module Blazer
|
2
|
+
module Adapters
|
3
|
+
class SodaAdapter < BaseAdapter
|
4
|
+
def run_statement(statement, comment)
|
5
|
+
require "json"
|
6
|
+
require "net/http"
|
7
|
+
require "uri"
|
8
|
+
|
9
|
+
columns = []
|
10
|
+
rows = []
|
11
|
+
error = nil
|
12
|
+
|
13
|
+
# remove comments manually
|
14
|
+
statement = statement.gsub(/--.+/, "")
|
15
|
+
# only supports single line /* */ comments
|
16
|
+
# regex not perfect, but should be good enough
|
17
|
+
statement = statement.gsub(/\/\*.+\*\//, "")
|
18
|
+
|
19
|
+
# remove trailing semicolon
|
20
|
+
statement = statement.sub(/;\s*\z/, "")
|
21
|
+
|
22
|
+
# remove whitespace
|
23
|
+
statement = statement.squish
|
24
|
+
|
25
|
+
uri = URI(settings["url"])
|
26
|
+
uri.query = URI.encode_www_form("$query" => statement)
|
27
|
+
|
28
|
+
req = Net::HTTP::Get.new(uri)
|
29
|
+
req["X-App-Token"] = settings["app_token"] if settings["app_token"]
|
30
|
+
|
31
|
+
options = {
|
32
|
+
use_ssl: uri.scheme == "https",
|
33
|
+
open_timeout: 3,
|
34
|
+
read_timeout: 30
|
35
|
+
}
|
36
|
+
|
37
|
+
begin
|
38
|
+
# use Net::HTTP instead of soda-ruby for types and better error messages
|
39
|
+
res = Net::HTTP.start(uri.hostname, uri.port, options) do |http|
|
40
|
+
http.request(req)
|
41
|
+
end
|
42
|
+
|
43
|
+
if res.is_a?(Net::HTTPSuccess)
|
44
|
+
body = JSON.parse(res.body)
|
45
|
+
|
46
|
+
columns = JSON.parse(res["x-soda2-fields"])
|
47
|
+
column_types = columns.zip(JSON.parse(res["x-soda2-types"])).to_h
|
48
|
+
|
49
|
+
columns.reject! { |f| f.start_with?(":@") }
|
50
|
+
# rows can be missing some keys in JSON, so need to map by column
|
51
|
+
rows = body.map { |r| columns.map { |c| r[c] } }
|
52
|
+
|
53
|
+
columns.each_with_index do |column, i|
|
54
|
+
# nothing to do for boolean
|
55
|
+
case column_types[column]
|
56
|
+
when "number"
|
57
|
+
# check if likely an integer column
|
58
|
+
if rows.all? { |r| r[i].to_i == r[i].to_f }
|
59
|
+
rows.each do |row|
|
60
|
+
row[i] = row[i].to_i
|
61
|
+
end
|
62
|
+
else
|
63
|
+
rows.each do |row|
|
64
|
+
row[i] = row[i].to_f
|
65
|
+
end
|
66
|
+
end
|
67
|
+
when "floating_timestamp"
|
68
|
+
# check if likely a date column
|
69
|
+
if rows.all? { |r| r[i].end_with?("T00:00:00.000") }
|
70
|
+
rows.each do |row|
|
71
|
+
row[i] = Date.parse(row[i])
|
72
|
+
end
|
73
|
+
else
|
74
|
+
utc = ActiveSupport::TimeZone["Etc/UTC"]
|
75
|
+
rows.each do |row|
|
76
|
+
row[i] = utc.parse(row[i])
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
else
|
82
|
+
error = JSON.parse(res.body)["message"] rescue "Bad response: #{res.code}"
|
83
|
+
end
|
84
|
+
rescue => e
|
85
|
+
error = e.message
|
86
|
+
end
|
87
|
+
|
88
|
+
[columns, rows, error]
|
89
|
+
end
|
90
|
+
|
91
|
+
def preview_statement
|
92
|
+
"SELECT * LIMIT 10"
|
93
|
+
end
|
94
|
+
|
95
|
+
def tables
|
96
|
+
["all"]
|
97
|
+
end
|
98
|
+
|
99
|
+
# https://dev.socrata.com/docs/datatypes/text.html
|
100
|
+
def quoting
|
101
|
+
:single_quote_escape
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Blazer
|
2
|
+
module Adapters
|
3
|
+
class SparkAdapter < HiveAdapter
|
4
|
+
def tables
|
5
|
+
client.execute("SHOW TABLES").map { |r| r["tableName"] }
|
6
|
+
end
|
7
|
+
|
8
|
+
# https://spark.apache.org/docs/latest/sql-ref-literals.html
|
9
|
+
def quoting
|
10
|
+
:backslash_escape
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|