mode 0.0.17 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -134
- data/lib/connect.rb +5 -5
- data/lib/mode.rb +12 -8
- data/lib/mode/api/form.rb +30 -8
- data/lib/mode/api/request.rb +34 -16
- data/lib/mode/commands/connect.rb +0 -8
- data/lib/mode/commands/import.rb +1 -1
- data/lib/mode/config.rb +9 -1
- data/lib/mode/connector/commands/select_report_run_dataset.rb +84 -0
- data/lib/mode/connector/commands/select_table_metadata.rb +113 -0
- data/lib/mode/connector/daemon.rb +3 -3
- data/lib/mode/connector/data_sources/base.rb +191 -0
- data/lib/mode/connector/databases/rdbms.rb +69 -0
- data/lib/mode/connector/dataset.rb +0 -1
- data/lib/mode/connector/dispatcher.rb +27 -0
- data/lib/mode/connector/poller.rb +2 -3
- data/lib/mode/connector/registrar.rb +74 -16
- data/lib/mode/connector/scheduler.rb +24 -15
- data/lib/mode/connector/selector.rb +1 -0
- data/lib/mode/connector/tables/rdbms.rb +130 -0
- data/lib/mode/version.rb +2 -2
- data/mode.gemspec +2 -0
- data/script/console.rb +11 -0
- data/spec/api/form_spec.rb +17 -8
- data/spec/api/request_spec.rb +3 -3
- data/spec/commands/connect_spec.rb +1 -10
- data/spec/config_spec.rb +1 -1
- data/spec/connector/commands/select_report_run_dataset_spec.rb +96 -0
- data/spec/connector/commands/select_table_metadata_spec.rb +115 -0
- data/spec/connector/{data_source_spec.rb → data_sources/base_spec.rb} +8 -8
- data/spec/connector/databases/rdbms_spec.rb +43 -0
- data/spec/connector/dispatcher_spec.rb +47 -0
- data/spec/connector/poller_spec.rb +2 -1
- data/spec/connector/registrar_spec.rb +111 -26
- data/spec/connector/scheduler_spec.rb +12 -31
- data/spec/connector/selector_spec.rb +1 -1
- data/spec/connector/tables/rdbms_spec.rb +85 -0
- metadata +49 -15
- data/lib/mode/connector/connect.rb +0 -11
- data/lib/mode/connector/data_source.rb +0 -171
- data/lib/mode/connector/message.rb +0 -31
- data/lib/mode/connector/processor.rb +0 -58
- data/lib/mode/connector/uploader.rb +0 -54
- data/spec/connector/message_spec.rb +0 -22
- data/spec/connector/processor_spec.rb +0 -93
- data/spec/connector/uploader_spec.rb +0 -57
@@ -0,0 +1,84 @@
|
|
1
|
+
module Mode
|
2
|
+
module Connector
|
3
|
+
module Commands
|
4
|
+
class SelectReportRunDataset
|
5
|
+
attr_reader :command, :data_sources
|
6
|
+
|
7
|
+
def initialize(command, data_sources)
|
8
|
+
@command = command
|
9
|
+
@data_sources = data_sources
|
10
|
+
end
|
11
|
+
|
12
|
+
def perform!
|
13
|
+
upload(dataset)
|
14
|
+
rescue => err
|
15
|
+
Mode::Logger.instance.error(
|
16
|
+
self.class.name, err.message, err.backtrace)
|
17
|
+
error(err.message, err.backtrace.join("\n"))
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def upload(dataset)
|
23
|
+
command.forms('edit').submit!(
|
24
|
+
:report_run => {
|
25
|
+
:dataset => {
|
26
|
+
:count => count(dataset),
|
27
|
+
:content => content(dataset),
|
28
|
+
:columns => JSON.generate(columns(dataset))
|
29
|
+
}
|
30
|
+
}
|
31
|
+
)
|
32
|
+
end
|
33
|
+
|
34
|
+
def error(message, detail = nil)
|
35
|
+
command.forms('edit').submit!(
|
36
|
+
:report_run => {
|
37
|
+
:error => {
|
38
|
+
:detail => detail,
|
39
|
+
:message => message
|
40
|
+
}
|
41
|
+
}
|
42
|
+
)
|
43
|
+
end
|
44
|
+
|
45
|
+
#
|
46
|
+
# Select Fields
|
47
|
+
#
|
48
|
+
|
49
|
+
def query
|
50
|
+
command.query
|
51
|
+
end
|
52
|
+
|
53
|
+
def data_source
|
54
|
+
@data_source ||= data_sources.find do |source|
|
55
|
+
source.name == command.embedded('data_source').name
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
#
|
60
|
+
# Dataset Fields
|
61
|
+
#
|
62
|
+
|
63
|
+
def dataset
|
64
|
+
@dataset ||=
|
65
|
+
Mode::Connector::Selector.new(query, data_source).perform!
|
66
|
+
end
|
67
|
+
|
68
|
+
def count(dataset)
|
69
|
+
dataset.count
|
70
|
+
end
|
71
|
+
|
72
|
+
def columns(dataset)
|
73
|
+
dataset.column_types.map do |name, type|
|
74
|
+
{:name => name, :type => type}
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def content(dataset)
|
79
|
+
Faraday::UploadIO.new(dataset.path, 'applicaton/octet-stream')
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
module Mode
|
2
|
+
module Connector
|
3
|
+
module Commands
|
4
|
+
class SelectTableMetadata
|
5
|
+
attr_reader :command, :data_sources
|
6
|
+
|
7
|
+
def initialize(command, data_sources)
|
8
|
+
@command = command
|
9
|
+
@data_sources = data_sources
|
10
|
+
end
|
11
|
+
|
12
|
+
def perform!
|
13
|
+
upload(table) && succeed
|
14
|
+
rescue => err
|
15
|
+
Mode::Logger.instance.error(
|
16
|
+
self.class.name, err.message, err.backtrace)
|
17
|
+
error(err.message, err.backtrace.join("\n"))
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
#
|
23
|
+
# Actions
|
24
|
+
#
|
25
|
+
|
26
|
+
def upload(table)
|
27
|
+
dataset = preview(table)
|
28
|
+
table_resource.forms('edit').submit!(
|
29
|
+
:table => {
|
30
|
+
:columns => columns(table),
|
31
|
+
:row_count => row_count(table),
|
32
|
+
:preview => {
|
33
|
+
:count => preview_count(dataset),
|
34
|
+
:columns => preview_columns(dataset),
|
35
|
+
:content => preview_content(dataset)
|
36
|
+
}
|
37
|
+
}
|
38
|
+
)
|
39
|
+
end
|
40
|
+
|
41
|
+
def error(message, detail = nil)
|
42
|
+
command.forms('edit').submit!(
|
43
|
+
:execution => {
|
44
|
+
:error => {
|
45
|
+
:detail => detail,
|
46
|
+
:message => message
|
47
|
+
}
|
48
|
+
}
|
49
|
+
)
|
50
|
+
end
|
51
|
+
|
52
|
+
def succeed
|
53
|
+
command.forms('succeed').submit!
|
54
|
+
end
|
55
|
+
|
56
|
+
#
|
57
|
+
# Metadata
|
58
|
+
#
|
59
|
+
|
60
|
+
def columns(table)
|
61
|
+
table.columns
|
62
|
+
end
|
63
|
+
|
64
|
+
def row_count(table)
|
65
|
+
table.row_count
|
66
|
+
end
|
67
|
+
|
68
|
+
#
|
69
|
+
# Previews
|
70
|
+
#
|
71
|
+
|
72
|
+
def preview(table)
|
73
|
+
table.preview
|
74
|
+
end
|
75
|
+
|
76
|
+
def preview_count(dataset)
|
77
|
+
dataset.count
|
78
|
+
end
|
79
|
+
|
80
|
+
def preview_columns(dataset)
|
81
|
+
dataset.column_types.map do |name, type|
|
82
|
+
{:name => name, :type => type}
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def preview_content(dataset)
|
87
|
+
Faraday::UploadIO.new(dataset.path, 'applicaton/octet-stream')
|
88
|
+
end
|
89
|
+
|
90
|
+
#
|
91
|
+
# Accessors
|
92
|
+
#
|
93
|
+
|
94
|
+
def table
|
95
|
+
table_schema = table_resource.schema
|
96
|
+
table_name = table_resource.name
|
97
|
+
|
98
|
+
@table ||= data_source.database.tables(table_schema, table_name)
|
99
|
+
end
|
100
|
+
|
101
|
+
def data_source
|
102
|
+
@data_source ||= data_sources.find do |source|
|
103
|
+
source.name == command.embedded('data_source').name
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def table_resource
|
108
|
+
@table_resources ||= command.embedded('table')
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
@@ -8,20 +8,20 @@ module Mode
|
|
8
8
|
@max_jobs = options[:max_jobs].to_i || 4
|
9
9
|
end
|
10
10
|
|
11
|
-
def start
|
11
|
+
def start
|
12
12
|
load_drivers
|
13
13
|
configure_api
|
14
14
|
scheduler.start!
|
15
15
|
rescue => err
|
16
16
|
Mode::Logger.instance.error(
|
17
|
-
|
17
|
+
self.class.name, err.message, err.backtrace)
|
18
18
|
end
|
19
19
|
|
20
20
|
def stop
|
21
21
|
scheduler.stop!
|
22
22
|
rescue => err
|
23
23
|
Mode::Logger.instance.error(
|
24
|
-
|
24
|
+
self.class.name, err.message, err.backtrace)
|
25
25
|
end
|
26
26
|
|
27
27
|
private
|
@@ -0,0 +1,191 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
|
3
|
+
module Mode
|
4
|
+
module Connector
|
5
|
+
module DataSources
|
6
|
+
class Base
|
7
|
+
attr_reader :name
|
8
|
+
attr_reader :props
|
9
|
+
|
10
|
+
attr_reader :connection
|
11
|
+
|
12
|
+
def initialize(name, props = {})
|
13
|
+
@name = name
|
14
|
+
@props = props
|
15
|
+
end
|
16
|
+
|
17
|
+
def adapter
|
18
|
+
props['adapter']
|
19
|
+
end
|
20
|
+
|
21
|
+
def username
|
22
|
+
props['username']
|
23
|
+
end
|
24
|
+
|
25
|
+
def password
|
26
|
+
props['password']
|
27
|
+
end
|
28
|
+
|
29
|
+
def host
|
30
|
+
props['host']
|
31
|
+
end
|
32
|
+
|
33
|
+
def port
|
34
|
+
props['port']
|
35
|
+
end
|
36
|
+
|
37
|
+
def database_name
|
38
|
+
props['database']
|
39
|
+
end
|
40
|
+
|
41
|
+
def ssl
|
42
|
+
props['ssl']
|
43
|
+
end
|
44
|
+
alias_method :ssl?, :ssl
|
45
|
+
|
46
|
+
def select(query, &block)
|
47
|
+
log_connection_query(query)
|
48
|
+
connection.dataset.fetch_rows(query, &block)
|
49
|
+
end
|
50
|
+
|
51
|
+
def database(options = {})
|
52
|
+
name = options[:database_name] || database_name
|
53
|
+
Mode::Connector::Databases::RDBMS.new(self, name)
|
54
|
+
end
|
55
|
+
|
56
|
+
def connection
|
57
|
+
@connection ||= Sequel.connect(connection_url, adapter_opts).tap do |conn|
|
58
|
+
conn.extension(:connection_validator)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def jdbc?
|
63
|
+
adapter.start_with?('jdbc')
|
64
|
+
end
|
65
|
+
|
66
|
+
def vertica?
|
67
|
+
['vertica', 'jdbc:vertica'].include?(adapter)
|
68
|
+
end
|
69
|
+
|
70
|
+
def redshift?
|
71
|
+
['redshift', 'jdbc:redshift'].include?(adapter)
|
72
|
+
end
|
73
|
+
|
74
|
+
def postgres?
|
75
|
+
['postgres', 'jdbc:postgresql'].include?(adapter)
|
76
|
+
end
|
77
|
+
|
78
|
+
def mysql?
|
79
|
+
['mysql', 'mysql2', 'jdbc:mysql'].include?(adapter)
|
80
|
+
end
|
81
|
+
|
82
|
+
def oracle?
|
83
|
+
['oracle', 'jdbc:oracle:thin', 'jdbc:oracle:oci8'].include?(adapter)
|
84
|
+
end
|
85
|
+
|
86
|
+
def sqlserver?
|
87
|
+
['tiny_tds', 'jdbc:sqlserver', 'jdbc:jtds:sqlserver'].include?(adapter)
|
88
|
+
end
|
89
|
+
|
90
|
+
def hive?
|
91
|
+
['hive', 'jdbc:hive2', 'jdbc:hive'].include?(adapter)
|
92
|
+
end
|
93
|
+
|
94
|
+
def has_information_schema?
|
95
|
+
!hive?
|
96
|
+
end
|
97
|
+
|
98
|
+
class << self
|
99
|
+
def build(name, props)
|
100
|
+
# inspect properties to build the correct klass
|
101
|
+
Mode::Connector::DataSources::Base.new(name, props)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
private
|
106
|
+
|
107
|
+
def adapter_opts
|
108
|
+
opts = {}
|
109
|
+
|
110
|
+
if redshift?
|
111
|
+
opts.merge!({
|
112
|
+
:client_min_messages => '',
|
113
|
+
:force_standard_strings => false
|
114
|
+
})
|
115
|
+
elsif postgres?
|
116
|
+
opts.merge!({
|
117
|
+
:sslmode => (ssl? ? ssl : 'prefer')
|
118
|
+
})
|
119
|
+
end
|
120
|
+
|
121
|
+
opts
|
122
|
+
end
|
123
|
+
|
124
|
+
def adapter_segment
|
125
|
+
case adapter
|
126
|
+
when 'jdbc:redshift'
|
127
|
+
'jdbc:postgresql'
|
128
|
+
# when 'jdbc:oracle:thin'
|
129
|
+
# jdbc:oracle:thin:scott/tiger@localhost:1521:orcl
|
130
|
+
else
|
131
|
+
adapter
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def port_segment
|
136
|
+
port.nil? ? nil : ":#{port}"
|
137
|
+
end
|
138
|
+
|
139
|
+
def password_segment
|
140
|
+
jdbc? ? jdbc_password_segment : standard_password_segment
|
141
|
+
end
|
142
|
+
|
143
|
+
def jdbc_password_segment
|
144
|
+
password.nil? ? nil : "&password=#{password}"
|
145
|
+
end
|
146
|
+
|
147
|
+
def standard_password_segment
|
148
|
+
password.nil? ? nil : ":#{password}"
|
149
|
+
end
|
150
|
+
|
151
|
+
def ssl_segment
|
152
|
+
jdbc? ? jdbc_ssl_segment : nil
|
153
|
+
end
|
154
|
+
|
155
|
+
def jdbc_ssl_segment
|
156
|
+
if ssl?
|
157
|
+
if mysql?
|
158
|
+
"&useSSL=true"
|
159
|
+
elsif sqlserver?
|
160
|
+
"&encrypt=true&trustServerCertificate=true"
|
161
|
+
elsif postgres? || vertica? || redshift?
|
162
|
+
"&ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory"
|
163
|
+
else
|
164
|
+
nil
|
165
|
+
end
|
166
|
+
else
|
167
|
+
nil
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
def connection_url
|
172
|
+
jdbc? ? jdbc_connection_url : standard_connection_url
|
173
|
+
end
|
174
|
+
|
175
|
+
def jdbc_connection_url
|
176
|
+
"#{adapter_segment}://#{host}#{port_segment}/#{database_name}?user=#{username}#{password_segment}#{ssl_segment}"
|
177
|
+
end
|
178
|
+
|
179
|
+
def standard_connection_url
|
180
|
+
"#{adapter_segment}://#{username}#{password_segment}@#{host}#{port_segment}/#{database_name}"
|
181
|
+
end
|
182
|
+
|
183
|
+
def log_connection_query(query)
|
184
|
+
return if query.nil?
|
185
|
+
Mode::Logger.instance.debug(
|
186
|
+
self.class.name, "QUERY", query.split("\n"))
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
module Mode
|
2
|
+
module Connector
|
3
|
+
module Databases
|
4
|
+
class RDBMS
|
5
|
+
attr_reader :data_source
|
6
|
+
attr_reader :database_name
|
7
|
+
|
8
|
+
def initialize(data_source, database_name)
|
9
|
+
@data_source = data_source
|
10
|
+
@database_name = database_name
|
11
|
+
end
|
12
|
+
|
13
|
+
#
|
14
|
+
# Returns the full dump of all columns in the database
|
15
|
+
#
|
16
|
+
def columns
|
17
|
+
columns = []
|
18
|
+
data_source.select(columns_query) do |column|
|
19
|
+
columns << column
|
20
|
+
end
|
21
|
+
columns
|
22
|
+
end
|
23
|
+
|
24
|
+
def tables(table_schema, table_name)
|
25
|
+
Mode::Connector::Tables::RDBMS.new(data_source, table_schema, table_name)
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def columns_query
|
31
|
+
%{
|
32
|
+
SELECT c.table_name,
|
33
|
+
c.table_schema,
|
34
|
+
c.column_name,
|
35
|
+
c.data_type,
|
36
|
+
CASE WHEN c.is_nullable = 'NO'
|
37
|
+
THEN FALSE
|
38
|
+
ELSE TRUE
|
39
|
+
END AS is_nullable,
|
40
|
+
CASE WHEN kc.name IS NULL
|
41
|
+
THEN FALSE
|
42
|
+
ELSE TRUE
|
43
|
+
END AS primary_key
|
44
|
+
FROM information_schema.columns c
|
45
|
+
LEFT JOIN (
|
46
|
+
SELECT kc.column_name AS name,
|
47
|
+
kc.table_catalog,
|
48
|
+
kc.table_schema,
|
49
|
+
kc.table_name
|
50
|
+
FROM information_schema.table_constraints c
|
51
|
+
LEFT JOIN information_schema.key_column_usage kc
|
52
|
+
ON c.table_catalog = kc.table_catalog
|
53
|
+
AND c.table_schema = kc.table_schema
|
54
|
+
AND c.table_name = kc.table_name
|
55
|
+
AND c.constraint_name = kc.constraint_name
|
56
|
+
WHERE c.constraint_type = 'PRIMARY KEY'
|
57
|
+
) kc
|
58
|
+
ON c.column_name = kc.name
|
59
|
+
AND c.table_catalog = kc.table_catalog
|
60
|
+
AND c.table_schema = kc.table_schema
|
61
|
+
AND c.table_name = kc.table_name
|
62
|
+
WHERE c.table_catalog = '#{database_name}'
|
63
|
+
ORDER BY c.table_name, c.table_schema, c.column_name
|
64
|
+
}
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|