mode 0.0.17 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -134
  3. data/lib/connect.rb +5 -5
  4. data/lib/mode.rb +12 -8
  5. data/lib/mode/api/form.rb +30 -8
  6. data/lib/mode/api/request.rb +34 -16
  7. data/lib/mode/commands/connect.rb +0 -8
  8. data/lib/mode/commands/import.rb +1 -1
  9. data/lib/mode/config.rb +9 -1
  10. data/lib/mode/connector/commands/select_report_run_dataset.rb +84 -0
  11. data/lib/mode/connector/commands/select_table_metadata.rb +113 -0
  12. data/lib/mode/connector/daemon.rb +3 -3
  13. data/lib/mode/connector/data_sources/base.rb +191 -0
  14. data/lib/mode/connector/databases/rdbms.rb +69 -0
  15. data/lib/mode/connector/dataset.rb +0 -1
  16. data/lib/mode/connector/dispatcher.rb +27 -0
  17. data/lib/mode/connector/poller.rb +2 -3
  18. data/lib/mode/connector/registrar.rb +74 -16
  19. data/lib/mode/connector/scheduler.rb +24 -15
  20. data/lib/mode/connector/selector.rb +1 -0
  21. data/lib/mode/connector/tables/rdbms.rb +130 -0
  22. data/lib/mode/version.rb +2 -2
  23. data/mode.gemspec +2 -0
  24. data/script/console.rb +11 -0
  25. data/spec/api/form_spec.rb +17 -8
  26. data/spec/api/request_spec.rb +3 -3
  27. data/spec/commands/connect_spec.rb +1 -10
  28. data/spec/config_spec.rb +1 -1
  29. data/spec/connector/commands/select_report_run_dataset_spec.rb +96 -0
  30. data/spec/connector/commands/select_table_metadata_spec.rb +115 -0
  31. data/spec/connector/{data_source_spec.rb → data_sources/base_spec.rb} +8 -8
  32. data/spec/connector/databases/rdbms_spec.rb +43 -0
  33. data/spec/connector/dispatcher_spec.rb +47 -0
  34. data/spec/connector/poller_spec.rb +2 -1
  35. data/spec/connector/registrar_spec.rb +111 -26
  36. data/spec/connector/scheduler_spec.rb +12 -31
  37. data/spec/connector/selector_spec.rb +1 -1
  38. data/spec/connector/tables/rdbms_spec.rb +85 -0
  39. metadata +49 -15
  40. data/lib/mode/connector/connect.rb +0 -11
  41. data/lib/mode/connector/data_source.rb +0 -171
  42. data/lib/mode/connector/message.rb +0 -31
  43. data/lib/mode/connector/processor.rb +0 -58
  44. data/lib/mode/connector/uploader.rb +0 -54
  45. data/spec/connector/message_spec.rb +0 -22
  46. data/spec/connector/processor_spec.rb +0 -93
  47. data/spec/connector/uploader_spec.rb +0 -57
@@ -2,7 +2,6 @@ module Mode
2
2
  module Connector
3
3
  class Dataset
4
4
  attr_reader :path
5
-
6
5
  attr_reader :count
7
6
  attr_reader :column_types
8
7
 
@@ -0,0 +1,27 @@
1
+ module Mode
2
+ module Connector
3
+ class Dispatcher # Rename to Dispatcher
4
+ attr_reader :resource
5
+ attr_reader :data_sources
6
+
7
+ Commands = Mode::Connector::Commands
8
+
9
+ def initialize(resource, data_sources)
10
+ @resource = resource
11
+ @data_sources = data_sources
12
+ end
13
+
14
+ def perform!
15
+ case resource.command['name']
16
+ when 'select:table:metadata'
17
+ Commands::SelectTableMetadata.new(resource, data_sources).perform!
18
+ when 'select:report_run:dataset'
19
+ Commands::SelectReportRunDataset.new(resource, data_sources).perform!
20
+ end
21
+ rescue => err
22
+ Mode::Logger.instance.error(
23
+ self.class.name, err.message, err.backtrace)
24
+ end
25
+ end
26
+ end
27
+ end
@@ -7,7 +7,7 @@ module Mode
7
7
  if messages.is_a?(Mode::API::Resource)
8
8
  messages.embedded('messages').each do |message|
9
9
  log_message(message)
10
- yield Mode::Connector::Message.new(message)
10
+ yield message
11
11
  end if messages.has_embedded?('messages')
12
12
  end
13
13
  end
@@ -19,8 +19,7 @@ module Mode
19
19
  end
20
20
 
21
21
  def log_message(message)
22
- Mode::Logger.instance.info(
23
- "Connector::Poller", message)
22
+ Mode::Logger.instance.info(self.class.name, message)
24
23
  end
25
24
  end
26
25
  end
@@ -1,33 +1,91 @@
1
1
  module Mode
2
2
  module Connector
3
3
  class Registrar
4
- attr_reader :config
4
+ attr_reader :data_sources
5
5
 
6
- def initialize(config)
7
- @config = config
6
+ def initialize(data_sources)
7
+ @data_sources = data_sources
8
8
  end
9
9
 
10
10
  def perform!
11
- Mode::API::Request.put(path,
12
- :data_source_connection => {
13
- :data_sources => data_sources
11
+ connection = register_connection
12
+ register_data_sources(connection)
13
+ end
14
+
15
+ private
16
+
17
+ def register_connection
18
+ Mode::API::Request.put(:data_source_connection, connection_params)
19
+ end
20
+
21
+ def register_data_sources(connection)
22
+ connection.embedded('data_sources').each do |resource|
23
+ data_source = data_sources.find{|l| l.name == resource.name }
24
+
25
+ if data_source && data_source.has_information_schema?
26
+ update_data_source_tables(data_source, resource)
27
+ end
28
+ end
29
+ end
30
+
31
+ def update_data_source_tables(data_source, resource)
32
+ resource.forms('edit').submit!(
33
+ :data_source => {
34
+ :tables => data_source_tables_params(data_source)
14
35
  }
15
36
  )
16
37
  end
17
38
 
18
- private
39
+ def connection_params
40
+ {
41
+ :data_source_connection => {
42
+ :data_sources => data_sources_params
43
+ }
44
+ }
45
+ end
46
+
47
+ def data_sources_params
48
+ data_sources.map { |source| data_source_params(source) }
49
+ end
50
+
51
+ def data_source_params(source)
52
+ { :name => source.name, :adapter => source.adapter }
53
+ end
54
+
55
+ def data_source_tables_params(source)
56
+ tables = []
57
+
58
+ current_table = nil
59
+ source.database.columns.each do |column|
60
+ table_schema = column[:table_schema]
61
+ table_name = column[:table_name]
19
62
 
20
- def data_sources
21
- config.data_sources.map do |source|
22
- {
23
- :name => source.name,
24
- :adapter => source.adapter
63
+ if current_table.nil?
64
+ current_table = { :columns => [] }
65
+ elsif current_table[:schema] != table_schema \
66
+ || current_table[:name] != table_name
67
+
68
+ tables << current_table.dup
69
+ current_table = { :columns => [] }
70
+ end
71
+
72
+ # Ensure Name
73
+ current_table[:schema] ||= table_schema
74
+ current_table[:name] ||= table_name
75
+
76
+ # Insert Column
77
+ current_table[:columns] << {
78
+ :name => column[:column_name],
79
+ :data_type => column[:data_type],
80
+ :is_nullable => column[:is_nullable],
81
+ :primary_key => column[:primary_key]
25
82
  }
26
83
  end
27
- end
28
-
29
- def path
30
- Mode::API::Request.data_source_connection_path
84
+
85
+ # Gotta catch 'em all!
86
+ tables << current_table.dup
87
+
88
+ tables
31
89
  end
32
90
  end
33
91
  end
@@ -14,8 +14,17 @@ module Mode
14
14
  end
15
15
 
16
16
  def start!
17
- data_sources.each(&:connection) # Keep outside the scheduler loop
18
- scheduler.interval('5s') { tick }
17
+ begin
18
+ register
19
+ data_sources.each(&:connection)
20
+ # Make sure to leave this outside the scheduler intervals
21
+ rescue => err
22
+ Mode::Logger.instance.error(
23
+ self.class.name, err.message, err.backtrace)
24
+ end
25
+
26
+ scheduler.interval('15m') { register }
27
+ scheduler.interval('5s') { process_messages }
19
28
  scheduler.join
20
29
  end
21
30
 
@@ -24,7 +33,7 @@ module Mode
24
33
  scheduler.stop # Stop polling
25
34
  }
26
35
 
27
- stopper.join # wait for threads to finish
36
+ stopper.join # wait for jobs to finish
28
37
  end
29
38
 
30
39
  def processors
@@ -37,30 +46,30 @@ module Mode
37
46
  max_jobs - processors.length
38
47
  end
39
48
 
40
- def tick
41
- if available_slots > 0
42
- poll_messages(:num_messages => available_slots) do |message|
43
- scheduler.in(0, :tag => 'processor') { tock(message) }
44
- end
45
- end
49
+ def register
50
+ Mode::Connector::Registrar.new(data_sources).perform!
46
51
  rescue => err
47
52
  Mode::Logger.instance.error(
48
- "Connector::Scheduler", err.message, err.backtrace)
53
+ self.class.name, err.message, err.backtrace)
49
54
  end
50
55
 
51
- def tock(message)
52
- process_message(message)
56
+ def process_messages
57
+ if available_slots > 0
58
+ poll_messages(:num_messages => available_slots) do |message|
59
+ scheduler.in(0, :tag => 'processor') { dispatch_message(message) }
60
+ end
61
+ end
53
62
  rescue => err
54
63
  Mode::Logger.instance.error(
55
- "Connector::Scheduler", err.message, err.backtrace)
64
+ self.class.name, err.message, err.backtrace)
56
65
  end
57
66
 
58
67
  def poll_messages(options = {}, &block)
59
68
  Mode::Connector::Poller.new.perform!(options, &block)
60
69
  end
61
70
 
62
- def process_message(message)
63
- Mode::Connector::Processor.new(message, data_sources).perform!
71
+ def dispatch_message(message)
72
+ Mode::Connector::Dispatcher.new(message, data_sources).perform!
64
73
  end
65
74
  end
66
75
  end
@@ -1,4 +1,5 @@
1
1
  require 'sequel'
2
+ require 'tmpdir'
2
3
 
3
4
  module Mode
4
5
  module Connector
@@ -0,0 +1,130 @@
1
+ module Mode
2
+ module Connector
3
+ module Tables
4
+ class RDBMS
5
+ attr_reader :data_source
6
+ attr_reader :table_schema
7
+ attr_reader :table_name
8
+
9
+ def initialize(data_source, table_schema, table_name)
10
+ @data_source = data_source
11
+ @table_schema = table_schema
12
+ @table_name = table_name
13
+ end
14
+
15
+ def row_count
16
+ return @row_count unless @row_count.nil?
17
+
18
+ data_source.select(row_count_query) do |row|
19
+ @row_count = row[:row_count]
20
+ end
21
+ @row_count
22
+ end
23
+
24
+ def columns
25
+ return @columns unless @columns.nil?
26
+
27
+ @columns = []
28
+ data_source.select(columns_query) do |row|
29
+ column = row.dup
30
+ column[:primary_key] = is_pkey?(row[:name]) ? true : false
31
+ column[:is_nullable] = row[:is_nullable] == 'YES' ? true : false
32
+
33
+ @columns << column
34
+ end
35
+ @columns
36
+ end
37
+
38
+ def preview
39
+ return @preview unless @preview.nil?
40
+ @preview = Mode::Connector::Selector.new(preview_query, data_source).perform!
41
+ end
42
+
43
+ #
44
+ # Primary Key Helpers
45
+ #
46
+
47
+ def pkey_columns
48
+ return @pkey_columns unless @pkey_columns.nil?
49
+
50
+ @pkey_columns = []
51
+ data_source.select(pkey_columns_query) do |row|
52
+ @pkey_columns << row
53
+ end
54
+ @pkey_columns
55
+ end
56
+
57
+ def is_pkey?(column_name)
58
+ pkey_columns.any?{|pkey| pkey[:name] == column_name}
59
+ end
60
+
61
+ def pkey_column_names
62
+ @pkey_column_names ||= pkey_columns.collect{|col| col[:name]}
63
+ end
64
+
65
+ private
66
+
67
+ def table_catalog
68
+ data_source.database_name
69
+ end
70
+
71
+ def full_table_name
72
+ "#{table_schema}.#{table_name}"
73
+ end
74
+
75
+ def row_count_query
76
+ %{
77
+ SELECT COUNT(1) as row_count
78
+ FROM #{full_table_name}
79
+ }
80
+ end
81
+
82
+ def preview_query
83
+ column_list = pkey_column_names.join(', ')
84
+ filter_list = pkey_column_names.map do |pkey_name|
85
+ "#{full_table_name}.#{pkey_name} = tmp.#{pkey_name}"
86
+ end.join(' AND ')
87
+
88
+ %{
89
+ SELECT *
90
+ FROM #{full_table_name}, (
91
+ SELECT #{column_list}
92
+ FROM #{full_table_name}
93
+ ORDER BY RANDOM()
94
+ LIMIT 50
95
+ ) tmp
96
+ WHERE #{filter_list}
97
+ }
98
+ end
99
+
100
+ def columns_query
101
+ %{
102
+ SELECT column_name AS name,
103
+ data_type,
104
+ is_nullable
105
+ FROM information_schema.columns
106
+ WHERE table_catalog = '#{table_catalog}'
107
+ AND table_schema = '#{table_schema}'
108
+ AND table_name = '#{table_name}'
109
+ }
110
+ end
111
+
112
+ def pkey_columns_query
113
+ %{
114
+ SELECT key_column_usage.column_name AS name
115
+ FROM information_schema.table_constraints constraints
116
+ LEFT JOIN information_schema.key_column_usage key_column_usage
117
+ ON constraints.table_catalog = key_column_usage.table_catalog
118
+ AND constraints.table_schema = key_column_usage.table_schema
119
+ AND constraints.table_name = key_column_usage.table_name
120
+ AND constraints.constraint_name = key_column_usage.constraint_name
121
+ WHERE constraints.constraint_type = 'PRIMARY KEY'
122
+ AND constraints.table_catalog = '#{table_catalog}'
123
+ AND constraints.table_schema = '#{table_schema}'
124
+ AND constraints.table_name = '#{table_name}'
125
+ }
126
+ end
127
+ end
128
+ end
129
+ end
130
+ end
@@ -1,3 +1,3 @@
1
1
  module Mode
2
- VERSION = "0.0.17"
3
- end
2
+ VERSION = "0.0.18"
3
+ end
@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
28
28
 
29
29
  # HTTP
30
30
  spec.add_runtime_dependency "faraday"
31
+ spec.add_runtime_dependency "faraday_middleware"
31
32
  spec.add_runtime_dependency "uri_template"
32
33
 
33
34
  # Connectivity
@@ -46,5 +47,6 @@ Gem::Specification.new do |spec|
46
47
  spec.add_development_dependency "profile"
47
48
  spec.add_development_dependency "rspec"
48
49
  spec.add_development_dependency "simplecov"
50
+ spec.add_development_dependency "pry"
49
51
  spec.add_development_dependency "warbler"
50
52
  end
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'pry'
4
+
5
+ $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
6
+
7
+ require 'mode'
8
+
9
+ Config = Mode::Config.new(Mode::Config.default_dir)
10
+
11
+ binding.pry
@@ -8,10 +8,16 @@ describe Mode::API::Link do
8
8
  'input' => {
9
9
  'resource' => {
10
10
  'name' => {
11
- 'type' => 'text',
12
- 'value' => ''
11
+ 'type' => 'text'
12
+ },
13
+
14
+ 'widgets[]' => {
15
+ 'name' => {
16
+ 'type' => 'text',
17
+ }
13
18
  },
14
- 'widgets' => {
19
+
20
+ 'widget_count' => {
15
21
  'type' => 'text',
16
22
  'value' => '500'
17
23
  }
@@ -33,17 +39,20 @@ describe Mode::API::Link do
33
39
 
34
40
  it "submits with params" do
35
41
  params = {
36
- 'resource' => {
37
- 'name' => 'test'
42
+ :resource => {
43
+ :name => 'test',
44
+ :widgets => [{
45
+ :name => 'widget1'
46
+ }]
38
47
  }
39
48
  }
40
49
 
41
50
  Mode::API::Request.should_receive(:post).with('/resources', {
42
51
  'resource' => {
43
52
  'name' => 'test',
44
- 'widgets' => '500'
45
- }
46
- }).and_return(:response)
53
+ 'widgets' => [{ 'name' => 'widget1' }],
54
+ 'widget_count' => '500' }
55
+ }, {}).and_return(:response)
47
56
 
48
57
  form = Mode::API::Form.new(content)
49
58
  form.submit!(params).should == :response