mode 0.0.17 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -134
- data/lib/connect.rb +5 -5
- data/lib/mode.rb +12 -8
- data/lib/mode/api/form.rb +30 -8
- data/lib/mode/api/request.rb +34 -16
- data/lib/mode/commands/connect.rb +0 -8
- data/lib/mode/commands/import.rb +1 -1
- data/lib/mode/config.rb +9 -1
- data/lib/mode/connector/commands/select_report_run_dataset.rb +84 -0
- data/lib/mode/connector/commands/select_table_metadata.rb +113 -0
- data/lib/mode/connector/daemon.rb +3 -3
- data/lib/mode/connector/data_sources/base.rb +191 -0
- data/lib/mode/connector/databases/rdbms.rb +69 -0
- data/lib/mode/connector/dataset.rb +0 -1
- data/lib/mode/connector/dispatcher.rb +27 -0
- data/lib/mode/connector/poller.rb +2 -3
- data/lib/mode/connector/registrar.rb +74 -16
- data/lib/mode/connector/scheduler.rb +24 -15
- data/lib/mode/connector/selector.rb +1 -0
- data/lib/mode/connector/tables/rdbms.rb +130 -0
- data/lib/mode/version.rb +2 -2
- data/mode.gemspec +2 -0
- data/script/console.rb +11 -0
- data/spec/api/form_spec.rb +17 -8
- data/spec/api/request_spec.rb +3 -3
- data/spec/commands/connect_spec.rb +1 -10
- data/spec/config_spec.rb +1 -1
- data/spec/connector/commands/select_report_run_dataset_spec.rb +96 -0
- data/spec/connector/commands/select_table_metadata_spec.rb +115 -0
- data/spec/connector/{data_source_spec.rb → data_sources/base_spec.rb} +8 -8
- data/spec/connector/databases/rdbms_spec.rb +43 -0
- data/spec/connector/dispatcher_spec.rb +47 -0
- data/spec/connector/poller_spec.rb +2 -1
- data/spec/connector/registrar_spec.rb +111 -26
- data/spec/connector/scheduler_spec.rb +12 -31
- data/spec/connector/selector_spec.rb +1 -1
- data/spec/connector/tables/rdbms_spec.rb +85 -0
- metadata +49 -15
- data/lib/mode/connector/connect.rb +0 -11
- data/lib/mode/connector/data_source.rb +0 -171
- data/lib/mode/connector/message.rb +0 -31
- data/lib/mode/connector/processor.rb +0 -58
- data/lib/mode/connector/uploader.rb +0 -54
- data/spec/connector/message_spec.rb +0 -22
- data/spec/connector/processor_spec.rb +0 -93
- data/spec/connector/uploader_spec.rb +0 -57
@@ -0,0 +1,27 @@
|
|
1
|
+
module Mode
|
2
|
+
module Connector
|
3
|
+
class Dispatcher # Rename to Dispatcher
|
4
|
+
attr_reader :resource
|
5
|
+
attr_reader :data_sources
|
6
|
+
|
7
|
+
Commands = Mode::Connector::Commands
|
8
|
+
|
9
|
+
def initialize(resource, data_sources)
|
10
|
+
@resource = resource
|
11
|
+
@data_sources = data_sources
|
12
|
+
end
|
13
|
+
|
14
|
+
def perform!
|
15
|
+
case resource.command['name']
|
16
|
+
when 'select:table:metadata'
|
17
|
+
Commands::SelectTableMetadata.new(resource, data_sources).perform!
|
18
|
+
when 'select:report_run:dataset'
|
19
|
+
Commands::SelectReportRunDataset.new(resource, data_sources).perform!
|
20
|
+
end
|
21
|
+
rescue => err
|
22
|
+
Mode::Logger.instance.error(
|
23
|
+
self.class.name, err.message, err.backtrace)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -7,7 +7,7 @@ module Mode
|
|
7
7
|
if messages.is_a?(Mode::API::Resource)
|
8
8
|
messages.embedded('messages').each do |message|
|
9
9
|
log_message(message)
|
10
|
-
yield
|
10
|
+
yield message
|
11
11
|
end if messages.has_embedded?('messages')
|
12
12
|
end
|
13
13
|
end
|
@@ -19,8 +19,7 @@ module Mode
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def log_message(message)
|
22
|
-
Mode::Logger.instance.info(
|
23
|
-
"Connector::Poller", message)
|
22
|
+
Mode::Logger.instance.info(self.class.name, message)
|
24
23
|
end
|
25
24
|
end
|
26
25
|
end
|
@@ -1,33 +1,91 @@
|
|
1
1
|
module Mode
|
2
2
|
module Connector
|
3
3
|
class Registrar
|
4
|
-
attr_reader :
|
4
|
+
attr_reader :data_sources
|
5
5
|
|
6
|
-
def initialize(
|
7
|
-
@
|
6
|
+
def initialize(data_sources)
|
7
|
+
@data_sources = data_sources
|
8
8
|
end
|
9
9
|
|
10
10
|
def perform!
|
11
|
-
|
12
|
-
|
13
|
-
|
11
|
+
connection = register_connection
|
12
|
+
register_data_sources(connection)
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def register_connection
|
18
|
+
Mode::API::Request.put(:data_source_connection, connection_params)
|
19
|
+
end
|
20
|
+
|
21
|
+
def register_data_sources(connection)
|
22
|
+
connection.embedded('data_sources').each do |resource|
|
23
|
+
data_source = data_sources.find{|l| l.name == resource.name }
|
24
|
+
|
25
|
+
if data_source && data_source.has_information_schema?
|
26
|
+
update_data_source_tables(data_source, resource)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def update_data_source_tables(data_source, resource)
|
32
|
+
resource.forms('edit').submit!(
|
33
|
+
:data_source => {
|
34
|
+
:tables => data_source_tables_params(data_source)
|
14
35
|
}
|
15
36
|
)
|
16
37
|
end
|
17
38
|
|
18
|
-
|
39
|
+
def connection_params
|
40
|
+
{
|
41
|
+
:data_source_connection => {
|
42
|
+
:data_sources => data_sources_params
|
43
|
+
}
|
44
|
+
}
|
45
|
+
end
|
46
|
+
|
47
|
+
def data_sources_params
|
48
|
+
data_sources.map { |source| data_source_params(source) }
|
49
|
+
end
|
50
|
+
|
51
|
+
def data_source_params(source)
|
52
|
+
{ :name => source.name, :adapter => source.adapter }
|
53
|
+
end
|
54
|
+
|
55
|
+
def data_source_tables_params(source)
|
56
|
+
tables = []
|
57
|
+
|
58
|
+
current_table = nil
|
59
|
+
source.database.columns.each do |column|
|
60
|
+
table_schema = column[:table_schema]
|
61
|
+
table_name = column[:table_name]
|
19
62
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
63
|
+
if current_table.nil?
|
64
|
+
current_table = { :columns => [] }
|
65
|
+
elsif current_table[:schema] != table_schema \
|
66
|
+
|| current_table[:name] != table_name
|
67
|
+
|
68
|
+
tables << current_table.dup
|
69
|
+
current_table = { :columns => [] }
|
70
|
+
end
|
71
|
+
|
72
|
+
# Ensure Name
|
73
|
+
current_table[:schema] ||= table_schema
|
74
|
+
current_table[:name] ||= table_name
|
75
|
+
|
76
|
+
# Insert Column
|
77
|
+
current_table[:columns] << {
|
78
|
+
:name => column[:column_name],
|
79
|
+
:data_type => column[:data_type],
|
80
|
+
:is_nullable => column[:is_nullable],
|
81
|
+
:primary_key => column[:primary_key]
|
25
82
|
}
|
26
83
|
end
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
84
|
+
|
85
|
+
# Gotta catch 'em all!
|
86
|
+
tables << current_table.dup
|
87
|
+
|
88
|
+
tables
|
31
89
|
end
|
32
90
|
end
|
33
91
|
end
|
@@ -14,8 +14,17 @@ module Mode
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def start!
|
17
|
-
|
18
|
-
|
17
|
+
begin
|
18
|
+
register
|
19
|
+
data_sources.each(&:connection)
|
20
|
+
# Make sure to leave this outside the scheduler intervals
|
21
|
+
rescue => err
|
22
|
+
Mode::Logger.instance.error(
|
23
|
+
self.class.name, err.message, err.backtrace)
|
24
|
+
end
|
25
|
+
|
26
|
+
scheduler.interval('15m') { register }
|
27
|
+
scheduler.interval('5s') { process_messages }
|
19
28
|
scheduler.join
|
20
29
|
end
|
21
30
|
|
@@ -24,7 +33,7 @@ module Mode
|
|
24
33
|
scheduler.stop # Stop polling
|
25
34
|
}
|
26
35
|
|
27
|
-
stopper.join # wait for
|
36
|
+
stopper.join # wait for jobs to finish
|
28
37
|
end
|
29
38
|
|
30
39
|
def processors
|
@@ -37,30 +46,30 @@ module Mode
|
|
37
46
|
max_jobs - processors.length
|
38
47
|
end
|
39
48
|
|
40
|
-
def
|
41
|
-
|
42
|
-
poll_messages(:num_messages => available_slots) do |message|
|
43
|
-
scheduler.in(0, :tag => 'processor') { tock(message) }
|
44
|
-
end
|
45
|
-
end
|
49
|
+
def register
|
50
|
+
Mode::Connector::Registrar.new(data_sources).perform!
|
46
51
|
rescue => err
|
47
52
|
Mode::Logger.instance.error(
|
48
|
-
|
53
|
+
self.class.name, err.message, err.backtrace)
|
49
54
|
end
|
50
55
|
|
51
|
-
def
|
52
|
-
|
56
|
+
def process_messages
|
57
|
+
if available_slots > 0
|
58
|
+
poll_messages(:num_messages => available_slots) do |message|
|
59
|
+
scheduler.in(0, :tag => 'processor') { dispatch_message(message) }
|
60
|
+
end
|
61
|
+
end
|
53
62
|
rescue => err
|
54
63
|
Mode::Logger.instance.error(
|
55
|
-
|
64
|
+
self.class.name, err.message, err.backtrace)
|
56
65
|
end
|
57
66
|
|
58
67
|
def poll_messages(options = {}, &block)
|
59
68
|
Mode::Connector::Poller.new.perform!(options, &block)
|
60
69
|
end
|
61
70
|
|
62
|
-
def
|
63
|
-
Mode::Connector::
|
71
|
+
def dispatch_message(message)
|
72
|
+
Mode::Connector::Dispatcher.new(message, data_sources).perform!
|
64
73
|
end
|
65
74
|
end
|
66
75
|
end
|
@@ -0,0 +1,130 @@
|
|
1
|
+
module Mode
|
2
|
+
module Connector
|
3
|
+
module Tables
|
4
|
+
class RDBMS
|
5
|
+
attr_reader :data_source
|
6
|
+
attr_reader :table_schema
|
7
|
+
attr_reader :table_name
|
8
|
+
|
9
|
+
def initialize(data_source, table_schema, table_name)
|
10
|
+
@data_source = data_source
|
11
|
+
@table_schema = table_schema
|
12
|
+
@table_name = table_name
|
13
|
+
end
|
14
|
+
|
15
|
+
def row_count
|
16
|
+
return @row_count unless @row_count.nil?
|
17
|
+
|
18
|
+
data_source.select(row_count_query) do |row|
|
19
|
+
@row_count = row[:row_count]
|
20
|
+
end
|
21
|
+
@row_count
|
22
|
+
end
|
23
|
+
|
24
|
+
def columns
|
25
|
+
return @columns unless @columns.nil?
|
26
|
+
|
27
|
+
@columns = []
|
28
|
+
data_source.select(columns_query) do |row|
|
29
|
+
column = row.dup
|
30
|
+
column[:primary_key] = is_pkey?(row[:name]) ? true : false
|
31
|
+
column[:is_nullable] = row[:is_nullable] == 'YES' ? true : false
|
32
|
+
|
33
|
+
@columns << column
|
34
|
+
end
|
35
|
+
@columns
|
36
|
+
end
|
37
|
+
|
38
|
+
def preview
|
39
|
+
return @preview unless @preview.nil?
|
40
|
+
@preview = Mode::Connector::Selector.new(preview_query, data_source).perform!
|
41
|
+
end
|
42
|
+
|
43
|
+
#
|
44
|
+
# Primary Key Helpers
|
45
|
+
#
|
46
|
+
|
47
|
+
def pkey_columns
|
48
|
+
return @pkey_columns unless @pkey_columns.nil?
|
49
|
+
|
50
|
+
@pkey_columns = []
|
51
|
+
data_source.select(pkey_columns_query) do |row|
|
52
|
+
@pkey_columns << row
|
53
|
+
end
|
54
|
+
@pkey_columns
|
55
|
+
end
|
56
|
+
|
57
|
+
def is_pkey?(column_name)
|
58
|
+
pkey_columns.any?{|pkey| pkey[:name] == column_name}
|
59
|
+
end
|
60
|
+
|
61
|
+
def pkey_column_names
|
62
|
+
@pkey_column_names ||= pkey_columns.collect{|col| col[:name]}
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def table_catalog
|
68
|
+
data_source.database_name
|
69
|
+
end
|
70
|
+
|
71
|
+
def full_table_name
|
72
|
+
"#{table_schema}.#{table_name}"
|
73
|
+
end
|
74
|
+
|
75
|
+
def row_count_query
|
76
|
+
%{
|
77
|
+
SELECT COUNT(1) as row_count
|
78
|
+
FROM #{full_table_name}
|
79
|
+
}
|
80
|
+
end
|
81
|
+
|
82
|
+
def preview_query
|
83
|
+
column_list = pkey_column_names.join(', ')
|
84
|
+
filter_list = pkey_column_names.map do |pkey_name|
|
85
|
+
"#{full_table_name}.#{pkey_name} = tmp.#{pkey_name}"
|
86
|
+
end.join(' AND ')
|
87
|
+
|
88
|
+
%{
|
89
|
+
SELECT *
|
90
|
+
FROM #{full_table_name}, (
|
91
|
+
SELECT #{column_list}
|
92
|
+
FROM #{full_table_name}
|
93
|
+
ORDER BY RANDOM()
|
94
|
+
LIMIT 50
|
95
|
+
) tmp
|
96
|
+
WHERE #{filter_list}
|
97
|
+
}
|
98
|
+
end
|
99
|
+
|
100
|
+
def columns_query
|
101
|
+
%{
|
102
|
+
SELECT column_name AS name,
|
103
|
+
data_type,
|
104
|
+
is_nullable
|
105
|
+
FROM information_schema.columns
|
106
|
+
WHERE table_catalog = '#{table_catalog}'
|
107
|
+
AND table_schema = '#{table_schema}'
|
108
|
+
AND table_name = '#{table_name}'
|
109
|
+
}
|
110
|
+
end
|
111
|
+
|
112
|
+
def pkey_columns_query
|
113
|
+
%{
|
114
|
+
SELECT key_column_usage.column_name AS name
|
115
|
+
FROM information_schema.table_constraints constraints
|
116
|
+
LEFT JOIN information_schema.key_column_usage key_column_usage
|
117
|
+
ON constraints.table_catalog = key_column_usage.table_catalog
|
118
|
+
AND constraints.table_schema = key_column_usage.table_schema
|
119
|
+
AND constraints.table_name = key_column_usage.table_name
|
120
|
+
AND constraints.constraint_name = key_column_usage.constraint_name
|
121
|
+
WHERE constraints.constraint_type = 'PRIMARY KEY'
|
122
|
+
AND constraints.table_catalog = '#{table_catalog}'
|
123
|
+
AND constraints.table_schema = '#{table_schema}'
|
124
|
+
AND constraints.table_name = '#{table_name}'
|
125
|
+
}
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
data/lib/mode/version.rb
CHANGED
@@ -1,3 +1,3 @@
|
|
1
1
|
module Mode
|
2
|
-
VERSION = "0.0.
|
3
|
-
end
|
2
|
+
VERSION = "0.0.18"
|
3
|
+
end
|
data/mode.gemspec
CHANGED
@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
|
|
28
28
|
|
29
29
|
# HTTP
|
30
30
|
spec.add_runtime_dependency "faraday"
|
31
|
+
spec.add_runtime_dependency "faraday_middleware"
|
31
32
|
spec.add_runtime_dependency "uri_template"
|
32
33
|
|
33
34
|
# Connectivity
|
@@ -46,5 +47,6 @@ Gem::Specification.new do |spec|
|
|
46
47
|
spec.add_development_dependency "profile"
|
47
48
|
spec.add_development_dependency "rspec"
|
48
49
|
spec.add_development_dependency "simplecov"
|
50
|
+
spec.add_development_dependency "pry"
|
49
51
|
spec.add_development_dependency "warbler"
|
50
52
|
end
|
data/script/console.rb
ADDED
data/spec/api/form_spec.rb
CHANGED
@@ -8,10 +8,16 @@ describe Mode::API::Link do
|
|
8
8
|
'input' => {
|
9
9
|
'resource' => {
|
10
10
|
'name' => {
|
11
|
-
'type' => 'text'
|
12
|
-
|
11
|
+
'type' => 'text'
|
12
|
+
},
|
13
|
+
|
14
|
+
'widgets[]' => {
|
15
|
+
'name' => {
|
16
|
+
'type' => 'text',
|
17
|
+
}
|
13
18
|
},
|
14
|
-
|
19
|
+
|
20
|
+
'widget_count' => {
|
15
21
|
'type' => 'text',
|
16
22
|
'value' => '500'
|
17
23
|
}
|
@@ -33,17 +39,20 @@ describe Mode::API::Link do
|
|
33
39
|
|
34
40
|
it "submits with params" do
|
35
41
|
params = {
|
36
|
-
|
37
|
-
|
42
|
+
:resource => {
|
43
|
+
:name => 'test',
|
44
|
+
:widgets => [{
|
45
|
+
:name => 'widget1'
|
46
|
+
}]
|
38
47
|
}
|
39
48
|
}
|
40
49
|
|
41
50
|
Mode::API::Request.should_receive(:post).with('/resources', {
|
42
51
|
'resource' => {
|
43
52
|
'name' => 'test',
|
44
|
-
'widgets' => '
|
45
|
-
|
46
|
-
}).and_return(:response)
|
53
|
+
'widgets' => [{ 'name' => 'widget1' }],
|
54
|
+
'widget_count' => '500' }
|
55
|
+
}, {}).and_return(:response)
|
47
56
|
|
48
57
|
form = Mode::API::Form.new(content)
|
49
58
|
form.submit!(params).should == :response
|