mode 0.0.17 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -134
  3. data/lib/connect.rb +5 -5
  4. data/lib/mode.rb +12 -8
  5. data/lib/mode/api/form.rb +30 -8
  6. data/lib/mode/api/request.rb +34 -16
  7. data/lib/mode/commands/connect.rb +0 -8
  8. data/lib/mode/commands/import.rb +1 -1
  9. data/lib/mode/config.rb +9 -1
  10. data/lib/mode/connector/commands/select_report_run_dataset.rb +84 -0
  11. data/lib/mode/connector/commands/select_table_metadata.rb +113 -0
  12. data/lib/mode/connector/daemon.rb +3 -3
  13. data/lib/mode/connector/data_sources/base.rb +191 -0
  14. data/lib/mode/connector/databases/rdbms.rb +69 -0
  15. data/lib/mode/connector/dataset.rb +0 -1
  16. data/lib/mode/connector/dispatcher.rb +27 -0
  17. data/lib/mode/connector/poller.rb +2 -3
  18. data/lib/mode/connector/registrar.rb +74 -16
  19. data/lib/mode/connector/scheduler.rb +24 -15
  20. data/lib/mode/connector/selector.rb +1 -0
  21. data/lib/mode/connector/tables/rdbms.rb +130 -0
  22. data/lib/mode/version.rb +2 -2
  23. data/mode.gemspec +2 -0
  24. data/script/console.rb +11 -0
  25. data/spec/api/form_spec.rb +17 -8
  26. data/spec/api/request_spec.rb +3 -3
  27. data/spec/commands/connect_spec.rb +1 -10
  28. data/spec/config_spec.rb +1 -1
  29. data/spec/connector/commands/select_report_run_dataset_spec.rb +96 -0
  30. data/spec/connector/commands/select_table_metadata_spec.rb +115 -0
  31. data/spec/connector/{data_source_spec.rb → data_sources/base_spec.rb} +8 -8
  32. data/spec/connector/databases/rdbms_spec.rb +43 -0
  33. data/spec/connector/dispatcher_spec.rb +47 -0
  34. data/spec/connector/poller_spec.rb +2 -1
  35. data/spec/connector/registrar_spec.rb +111 -26
  36. data/spec/connector/scheduler_spec.rb +12 -31
  37. data/spec/connector/selector_spec.rb +1 -1
  38. data/spec/connector/tables/rdbms_spec.rb +85 -0
  39. metadata +49 -15
  40. data/lib/mode/connector/connect.rb +0 -11
  41. data/lib/mode/connector/data_source.rb +0 -171
  42. data/lib/mode/connector/message.rb +0 -31
  43. data/lib/mode/connector/processor.rb +0 -58
  44. data/lib/mode/connector/uploader.rb +0 -54
  45. data/spec/connector/message_spec.rb +0 -22
  46. data/spec/connector/processor_spec.rb +0 -93
  47. data/spec/connector/uploader_spec.rb +0 -57
@@ -0,0 +1,84 @@
1
+ module Mode
2
+ module Connector
3
+ module Commands
4
+ class SelectReportRunDataset
5
+ attr_reader :command, :data_sources
6
+
7
+ def initialize(command, data_sources)
8
+ @command = command
9
+ @data_sources = data_sources
10
+ end
11
+
12
+ def perform!
13
+ upload(dataset)
14
+ rescue => err
15
+ Mode::Logger.instance.error(
16
+ self.class.name, err.message, err.backtrace)
17
+ error(err.message, err.backtrace.join("\n"))
18
+ end
19
+
20
+ private
21
+
22
+ def upload(dataset)
23
+ command.forms('edit').submit!(
24
+ :report_run => {
25
+ :dataset => {
26
+ :count => count(dataset),
27
+ :content => content(dataset),
28
+ :columns => JSON.generate(columns(dataset))
29
+ }
30
+ }
31
+ )
32
+ end
33
+
34
+ def error(message, detail = nil)
35
+ command.forms('edit').submit!(
36
+ :report_run => {
37
+ :error => {
38
+ :detail => detail,
39
+ :message => message
40
+ }
41
+ }
42
+ )
43
+ end
44
+
45
+ #
46
+ # Select Fields
47
+ #
48
+
49
+ def query
50
+ command.query
51
+ end
52
+
53
+ def data_source
54
+ @data_source ||= data_sources.find do |source|
55
+ source.name == command.embedded('data_source').name
56
+ end
57
+ end
58
+
59
+ #
60
+ # Dataset Fields
61
+ #
62
+
63
+ def dataset
64
+ @dataset ||=
65
+ Mode::Connector::Selector.new(query, data_source).perform!
66
+ end
67
+
68
+ def count(dataset)
69
+ dataset.count
70
+ end
71
+
72
+ def columns(dataset)
73
+ dataset.column_types.map do |name, type|
74
+ {:name => name, :type => type}
75
+ end
76
+ end
77
+
78
+ def content(dataset)
79
+ Faraday::UploadIO.new(dataset.path, 'applicaton/octet-stream')
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,113 @@
1
+ module Mode
2
+ module Connector
3
+ module Commands
4
+ class SelectTableMetadata
5
+ attr_reader :command, :data_sources
6
+
7
+ def initialize(command, data_sources)
8
+ @command = command
9
+ @data_sources = data_sources
10
+ end
11
+
12
+ def perform!
13
+ upload(table) && succeed
14
+ rescue => err
15
+ Mode::Logger.instance.error(
16
+ self.class.name, err.message, err.backtrace)
17
+ error(err.message, err.backtrace.join("\n"))
18
+ end
19
+
20
+ private
21
+
22
+ #
23
+ # Actions
24
+ #
25
+
26
+ def upload(table)
27
+ dataset = preview(table)
28
+ table_resource.forms('edit').submit!(
29
+ :table => {
30
+ :columns => columns(table),
31
+ :row_count => row_count(table),
32
+ :preview => {
33
+ :count => preview_count(dataset),
34
+ :columns => preview_columns(dataset),
35
+ :content => preview_content(dataset)
36
+ }
37
+ }
38
+ )
39
+ end
40
+
41
+ def error(message, detail = nil)
42
+ command.forms('edit').submit!(
43
+ :execution => {
44
+ :error => {
45
+ :detail => detail,
46
+ :message => message
47
+ }
48
+ }
49
+ )
50
+ end
51
+
52
+ def succeed
53
+ command.forms('succeed').submit!
54
+ end
55
+
56
+ #
57
+ # Metadata
58
+ #
59
+
60
+ def columns(table)
61
+ table.columns
62
+ end
63
+
64
+ def row_count(table)
65
+ table.row_count
66
+ end
67
+
68
+ #
69
+ # Previews
70
+ #
71
+
72
+ def preview(table)
73
+ table.preview
74
+ end
75
+
76
+ def preview_count(dataset)
77
+ dataset.count
78
+ end
79
+
80
+ def preview_columns(dataset)
81
+ dataset.column_types.map do |name, type|
82
+ {:name => name, :type => type}
83
+ end
84
+ end
85
+
86
+ def preview_content(dataset)
87
+ Faraday::UploadIO.new(dataset.path, 'applicaton/octet-stream')
88
+ end
89
+
90
+ #
91
+ # Accessors
92
+ #
93
+
94
+ def table
95
+ table_schema = table_resource.schema
96
+ table_name = table_resource.name
97
+
98
+ @table ||= data_source.database.tables(table_schema, table_name)
99
+ end
100
+
101
+ def data_source
102
+ @data_source ||= data_sources.find do |source|
103
+ source.name == command.embedded('data_source').name
104
+ end
105
+ end
106
+
107
+ def table_resource
108
+ @table_resources ||= command.embedded('table')
109
+ end
110
+ end
111
+ end
112
+ end
113
+ end
@@ -8,20 +8,20 @@ module Mode
8
8
  @max_jobs = options[:max_jobs].to_i || 4
9
9
  end
10
10
 
11
- def start
11
+ def start
12
12
  load_drivers
13
13
  configure_api
14
14
  scheduler.start!
15
15
  rescue => err
16
16
  Mode::Logger.instance.error(
17
- "Connector::Runner", err.message, err.backtrace)
17
+ self.class.name, err.message, err.backtrace)
18
18
  end
19
19
 
20
20
  def stop
21
21
  scheduler.stop!
22
22
  rescue => err
23
23
  Mode::Logger.instance.error(
24
- "Connector::Runner", err.message, err.backtrace)
24
+ self.class.name, err.message, err.backtrace)
25
25
  end
26
26
 
27
27
  private
@@ -0,0 +1,191 @@
1
+ require 'sequel'
2
+
3
+ module Mode
4
+ module Connector
5
+ module DataSources
6
+ class Base
7
+ attr_reader :name
8
+ attr_reader :props
9
+
10
+ attr_reader :connection
11
+
12
+ def initialize(name, props = {})
13
+ @name = name
14
+ @props = props
15
+ end
16
+
17
+ def adapter
18
+ props['adapter']
19
+ end
20
+
21
+ def username
22
+ props['username']
23
+ end
24
+
25
+ def password
26
+ props['password']
27
+ end
28
+
29
+ def host
30
+ props['host']
31
+ end
32
+
33
+ def port
34
+ props['port']
35
+ end
36
+
37
+ def database_name
38
+ props['database']
39
+ end
40
+
41
+ def ssl
42
+ props['ssl']
43
+ end
44
+ alias_method :ssl?, :ssl
45
+
46
+ def select(query, &block)
47
+ log_connection_query(query)
48
+ connection.dataset.fetch_rows(query, &block)
49
+ end
50
+
51
+ def database(options = {})
52
+ name = options[:database_name] || database_name
53
+ Mode::Connector::Databases::RDBMS.new(self, name)
54
+ end
55
+
56
+ def connection
57
+ @connection ||= Sequel.connect(connection_url, adapter_opts).tap do |conn|
58
+ conn.extension(:connection_validator)
59
+ end
60
+ end
61
+
62
+ def jdbc?
63
+ adapter.start_with?('jdbc')
64
+ end
65
+
66
+ def vertica?
67
+ ['vertica', 'jdbc:vertica'].include?(adapter)
68
+ end
69
+
70
+ def redshift?
71
+ ['redshift', 'jdbc:redshift'].include?(adapter)
72
+ end
73
+
74
+ def postgres?
75
+ ['postgres', 'jdbc:postgresql'].include?(adapter)
76
+ end
77
+
78
+ def mysql?
79
+ ['mysql', 'mysql2', 'jdbc:mysql'].include?(adapter)
80
+ end
81
+
82
+ def oracle?
83
+ ['oracle', 'jdbc:oracle:thin', 'jdbc:oracle:oci8'].include?(adapter)
84
+ end
85
+
86
+ def sqlserver?
87
+ ['tiny_tds', 'jdbc:sqlserver', 'jdbc:jtds:sqlserver'].include?(adapter)
88
+ end
89
+
90
+ def hive?
91
+ ['hive', 'jdbc:hive2', 'jdbc:hive'].include?(adapter)
92
+ end
93
+
94
+ def has_information_schema?
95
+ !hive?
96
+ end
97
+
98
+ class << self
99
+ def build(name, props)
100
+ # inspect properties to build the correct klass
101
+ Mode::Connector::DataSources::Base.new(name, props)
102
+ end
103
+ end
104
+
105
+ private
106
+
107
+ def adapter_opts
108
+ opts = {}
109
+
110
+ if redshift?
111
+ opts.merge!({
112
+ :client_min_messages => '',
113
+ :force_standard_strings => false
114
+ })
115
+ elsif postgres?
116
+ opts.merge!({
117
+ :sslmode => (ssl? ? ssl : 'prefer')
118
+ })
119
+ end
120
+
121
+ opts
122
+ end
123
+
124
+ def adapter_segment
125
+ case adapter
126
+ when 'jdbc:redshift'
127
+ 'jdbc:postgresql'
128
+ # when 'jdbc:oracle:thin'
129
+ # jdbc:oracle:thin:scott/tiger@localhost:1521:orcl
130
+ else
131
+ adapter
132
+ end
133
+ end
134
+
135
+ def port_segment
136
+ port.nil? ? nil : ":#{port}"
137
+ end
138
+
139
+ def password_segment
140
+ jdbc? ? jdbc_password_segment : standard_password_segment
141
+ end
142
+
143
+ def jdbc_password_segment
144
+ password.nil? ? nil : "&password=#{password}"
145
+ end
146
+
147
+ def standard_password_segment
148
+ password.nil? ? nil : ":#{password}"
149
+ end
150
+
151
+ def ssl_segment
152
+ jdbc? ? jdbc_ssl_segment : nil
153
+ end
154
+
155
+ def jdbc_ssl_segment
156
+ if ssl?
157
+ if mysql?
158
+ "&useSSL=true"
159
+ elsif sqlserver?
160
+ "&encrypt=true&trustServerCertificate=true"
161
+ elsif postgres? || vertica? || redshift?
162
+ "&ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory"
163
+ else
164
+ nil
165
+ end
166
+ else
167
+ nil
168
+ end
169
+ end
170
+
171
+ def connection_url
172
+ jdbc? ? jdbc_connection_url : standard_connection_url
173
+ end
174
+
175
+ def jdbc_connection_url
176
+ "#{adapter_segment}://#{host}#{port_segment}/#{database_name}?user=#{username}#{password_segment}#{ssl_segment}"
177
+ end
178
+
179
+ def standard_connection_url
180
+ "#{adapter_segment}://#{username}#{password_segment}@#{host}#{port_segment}/#{database_name}"
181
+ end
182
+
183
+ def log_connection_query(query)
184
+ return if query.nil?
185
+ Mode::Logger.instance.debug(
186
+ self.class.name, "QUERY", query.split("\n"))
187
+ end
188
+ end
189
+ end
190
+ end
191
+ end
@@ -0,0 +1,69 @@
1
+ module Mode
2
+ module Connector
3
+ module Databases
4
+ class RDBMS
5
+ attr_reader :data_source
6
+ attr_reader :database_name
7
+
8
+ def initialize(data_source, database_name)
9
+ @data_source = data_source
10
+ @database_name = database_name
11
+ end
12
+
13
+ #
14
+ # Returns the full dump of all columns in the database
15
+ #
16
+ def columns
17
+ columns = []
18
+ data_source.select(columns_query) do |column|
19
+ columns << column
20
+ end
21
+ columns
22
+ end
23
+
24
+ def tables(table_schema, table_name)
25
+ Mode::Connector::Tables::RDBMS.new(data_source, table_schema, table_name)
26
+ end
27
+
28
+ private
29
+
30
+ def columns_query
31
+ %{
32
+ SELECT c.table_name,
33
+ c.table_schema,
34
+ c.column_name,
35
+ c.data_type,
36
+ CASE WHEN c.is_nullable = 'NO'
37
+ THEN FALSE
38
+ ELSE TRUE
39
+ END AS is_nullable,
40
+ CASE WHEN kc.name IS NULL
41
+ THEN FALSE
42
+ ELSE TRUE
43
+ END AS primary_key
44
+ FROM information_schema.columns c
45
+ LEFT JOIN (
46
+ SELECT kc.column_name AS name,
47
+ kc.table_catalog,
48
+ kc.table_schema,
49
+ kc.table_name
50
+ FROM information_schema.table_constraints c
51
+ LEFT JOIN information_schema.key_column_usage kc
52
+ ON c.table_catalog = kc.table_catalog
53
+ AND c.table_schema = kc.table_schema
54
+ AND c.table_name = kc.table_name
55
+ AND c.constraint_name = kc.constraint_name
56
+ WHERE c.constraint_type = 'PRIMARY KEY'
57
+ ) kc
58
+ ON c.column_name = kc.name
59
+ AND c.table_catalog = kc.table_catalog
60
+ AND c.table_schema = kc.table_schema
61
+ AND c.table_name = kc.table_name
62
+ WHERE c.table_catalog = '#{database_name}'
63
+ ORDER BY c.table_name, c.table_schema, c.column_name
64
+ }
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end