mode 0.0.17 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -134
  3. data/lib/connect.rb +5 -5
  4. data/lib/mode.rb +12 -8
  5. data/lib/mode/api/form.rb +30 -8
  6. data/lib/mode/api/request.rb +34 -16
  7. data/lib/mode/commands/connect.rb +0 -8
  8. data/lib/mode/commands/import.rb +1 -1
  9. data/lib/mode/config.rb +9 -1
  10. data/lib/mode/connector/commands/select_report_run_dataset.rb +84 -0
  11. data/lib/mode/connector/commands/select_table_metadata.rb +113 -0
  12. data/lib/mode/connector/daemon.rb +3 -3
  13. data/lib/mode/connector/data_sources/base.rb +191 -0
  14. data/lib/mode/connector/databases/rdbms.rb +69 -0
  15. data/lib/mode/connector/dataset.rb +0 -1
  16. data/lib/mode/connector/dispatcher.rb +27 -0
  17. data/lib/mode/connector/poller.rb +2 -3
  18. data/lib/mode/connector/registrar.rb +74 -16
  19. data/lib/mode/connector/scheduler.rb +24 -15
  20. data/lib/mode/connector/selector.rb +1 -0
  21. data/lib/mode/connector/tables/rdbms.rb +130 -0
  22. data/lib/mode/version.rb +2 -2
  23. data/mode.gemspec +2 -0
  24. data/script/console.rb +11 -0
  25. data/spec/api/form_spec.rb +17 -8
  26. data/spec/api/request_spec.rb +3 -3
  27. data/spec/commands/connect_spec.rb +1 -10
  28. data/spec/config_spec.rb +1 -1
  29. data/spec/connector/commands/select_report_run_dataset_spec.rb +96 -0
  30. data/spec/connector/commands/select_table_metadata_spec.rb +115 -0
  31. data/spec/connector/{data_source_spec.rb → data_sources/base_spec.rb} +8 -8
  32. data/spec/connector/databases/rdbms_spec.rb +43 -0
  33. data/spec/connector/dispatcher_spec.rb +47 -0
  34. data/spec/connector/poller_spec.rb +2 -1
  35. data/spec/connector/registrar_spec.rb +111 -26
  36. data/spec/connector/scheduler_spec.rb +12 -31
  37. data/spec/connector/selector_spec.rb +1 -1
  38. data/spec/connector/tables/rdbms_spec.rb +85 -0
  39. metadata +49 -15
  40. data/lib/mode/connector/connect.rb +0 -11
  41. data/lib/mode/connector/data_source.rb +0 -171
  42. data/lib/mode/connector/message.rb +0 -31
  43. data/lib/mode/connector/processor.rb +0 -58
  44. data/lib/mode/connector/uploader.rb +0 -54
  45. data/spec/connector/message_spec.rb +0 -22
  46. data/spec/connector/processor_spec.rb +0 -93
  47. data/spec/connector/uploader_spec.rb +0 -57
@@ -3,7 +3,7 @@ require 'spec_helper'
3
3
  describe Mode::Connector::Scheduler do
4
4
  let(:data_sources) {
5
5
  [
6
- Mode::Connector::DataSource.new('dev', {
6
+ Mode::Connector::DataSources::Base.new('dev', {
7
7
  'adapter' => 'postgres',
8
8
  'host' => 'localhost',
9
9
  'username' => 'postgres',
@@ -12,22 +12,22 @@ describe Mode::Connector::Scheduler do
12
12
  })
13
13
  ]
14
14
  }
15
-
15
+
16
16
  before do
17
17
  initialize_logger
18
+
19
+ conn = double(:conn, :extension => true)
20
+ Sequel.stub(:connect).and_return(conn)
18
21
  end
19
22
 
20
23
  it 'starts and stops a scheduler' do
21
24
  rufus = double(:rufus)
22
25
  Rufus::Scheduler.should_receive(:new).and_return(rufus)
23
26
 
24
- rufus.should_receive(:interval).and_return(true)
27
+ rufus.should_receive(:interval).twice.and_return(true)
25
28
  rufus.should_receive(:join).and_return(true)
26
29
  rufus.should_receive(:stop).and_return(true)
27
30
 
28
- conn = double(:conn, :extension => true)
29
- Sequel.should_receive(:connect).and_return(conn)
30
-
31
31
  scheduler = Mode::Connector::Scheduler.new(data_sources)
32
32
 
33
33
  scheduler.start!
@@ -46,34 +46,15 @@ describe Mode::Connector::Scheduler do
46
46
  end
47
47
  end
48
48
 
49
- it 'processes a message' do
50
- processor = double(:processor)
51
- processor.should_receive(:perform!).and_return(true)
52
- Mode::Connector::Processor.should_receive(:new).with(:message, data_sources).and_return(processor)
53
-
54
- scheduler = Mode::Connector::Scheduler.new(data_sources)
55
-
56
- scheduler.send(:process_message, :message)
57
- end
58
-
59
- it "polls and processes messages" do
60
- scheduler = Mode::Connector::Scheduler.new(data_sources)
61
-
62
- message = double(:message, :name => 'name', :query => 'SELECT 1')
63
- scheduler.should_receive(:process_message).with(message).and_return(true)
64
-
65
- scheduler.send(:tock, message)
66
- end
67
-
68
- it "catches errors during tocks" do
49
+ it "catches errors during messages processing" do
69
50
  scheduler = Mode::Connector::Scheduler.new(data_sources)
70
51
 
71
- message = "Broken tocking!"
72
- scheduler.should_receive(:process_message).and_raise(StandardError.new(message))
52
+ message = "Broken dispatching!"
53
+ scheduler.should_receive(:poll_messages).and_raise(StandardError.new(message))
73
54
 
74
55
  Mode::Logger.instance.should_receive(:error)
75
56
 
76
- scheduler.send(:tock, message)
57
+ scheduler.send(:process_messages)
77
58
  end
78
59
 
79
60
  it "polls if there are available threads in the pool" do
@@ -83,7 +64,7 @@ describe Mode::Connector::Scheduler do
83
64
  scheduler.should_receive(:poll_messages).and_yield(message)
84
65
  scheduler.scheduler.should_receive(:in).and_return(true)
85
66
 
86
- scheduler.send(:tick)
67
+ scheduler.send(:process_messages)
87
68
  end
88
69
 
89
70
  it "doesn't poll unless there are available threads in the pool" do
@@ -91,6 +72,6 @@ describe Mode::Connector::Scheduler do
91
72
 
92
73
  scheduler.should_not_receive(:tock)
93
74
 
94
- scheduler.send(:tick)
75
+ scheduler.send(:process_messages)
95
76
  end
96
77
  end
@@ -6,7 +6,7 @@ describe Mode::Connector::Selector do
6
6
  let(:outpath) { File.join(tmpdir, 'data.csv') }
7
7
 
8
8
  let(:data_source) {
9
- Mode::Connector::DataSource.new('testdb', {
9
+ Mode::Connector::DataSources::Base.new('testdb', {
10
10
  'adapter' => 'postgres',
11
11
  'host' => 'localhost',
12
12
  'username' => 'postgres',
@@ -0,0 +1,85 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Tables::RDBMS do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+ let(:config) { Mode::Config.init(tmpdir) }
6
+
7
+ let(:data_source) {
8
+ Mode::Connector::DataSources::Base.new('testdb', {
9
+ 'adapter' => 'postgres',
10
+ 'host' => 'localhost',
11
+ 'username' => 'postgres',
12
+ 'database' => 'warehouse',
13
+ 'password' => nil
14
+ })
15
+ }
16
+
17
+ before do
18
+ initialize_logger
19
+ end
20
+
21
+ it "initializes" do
22
+ table = Mode::Connector::Tables::RDBMS.new(data_source, 'warehouse', 'accounts')
23
+
24
+ table.data_source.should == data_source
25
+ table.table_schema.should == 'warehouse'
26
+ table.table_name.should == 'accounts'
27
+ end
28
+
29
+ it "has a row count" do
30
+ table = Mode::Connector::Tables::RDBMS.new(data_source, 'warehouse', 'accounts')
31
+
32
+ data_source.should_receive(:select).and_yield({:row_count => 20})
33
+
34
+ table.row_count.should == 20
35
+ end
36
+
37
+ it "has column metadata" do
38
+ table = Mode::Connector::Tables::RDBMS.new(data_source, 'warehouse', 'accounts')
39
+
40
+ column = {
41
+ :name => 'account_id',
42
+ :data_type => 'integer',
43
+ :primary_key => true,
44
+ :is_nullable => false
45
+ }
46
+
47
+ table.stub(:pkey_columns).and_return([column])
48
+ data_source.should_receive(:select).and_yield(column)
49
+
50
+ table.columns.should == [column]
51
+ end
52
+
53
+ it "has primary key columns" do
54
+ table = Mode::Connector::Tables::RDBMS.new(data_source, 'warehouse', 'accounts')
55
+
56
+ column = {
57
+ :name => 'account_id',
58
+ :data_type => 'integer',
59
+ :primary_key => true,
60
+ :is_nullable => false
61
+ }
62
+
63
+ data_source.should_receive(:select).and_yield(column)
64
+
65
+ table.pkey_columns.should == [column]
66
+ end
67
+
68
+ it "generates a preview" do
69
+ table = Mode::Connector::Tables::RDBMS.new(data_source, 'warehouse', 'accounts')
70
+
71
+ column = {
72
+ :name => 'account_id',
73
+ :data_type => 'integer',
74
+ :primary_key => true,
75
+ :is_nullable => false
76
+ }
77
+
78
+ table.stub(:pkey_columns).and_return([column])
79
+
80
+ selector = double(:selector, :perform! => :preview)
81
+ Mode::Connector::Selector.should_receive(:new).and_return(selector)
82
+
83
+ table.preview.should == :preview
84
+ end
85
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mode
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.17
4
+ version: 0.0.18
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mode Analytics
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-02-06 00:00:00.000000000 Z
11
+ date: 2014-03-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor
@@ -80,6 +80,20 @@ dependencies:
80
80
  version: '0'
81
81
  prerelease: false
82
82
  type: :runtime
83
+ - !ruby/object:Gem::Dependency
84
+ name: faraday_middleware
85
+ version_requirements: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - '>='
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ requirement: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - '>='
93
+ - !ruby/object:Gem::Version
94
+ version: '0'
95
+ prerelease: false
96
+ type: :runtime
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: uri_template
85
99
  version_requirements: !ruby/object:Gem::Requirement
@@ -206,6 +220,20 @@ dependencies:
206
220
  version: '0'
207
221
  prerelease: false
208
222
  type: :development
223
+ - !ruby/object:Gem::Dependency
224
+ name: pry
225
+ version_requirements: !ruby/object:Gem::Requirement
226
+ requirements:
227
+ - - '>='
228
+ - !ruby/object:Gem::Version
229
+ version: '0'
230
+ requirement: !ruby/object:Gem::Requirement
231
+ requirements:
232
+ - - '>='
233
+ - !ruby/object:Gem::Version
234
+ version: '0'
235
+ prerelease: false
236
+ type: :development
209
237
  - !ruby/object:Gem::Dependency
210
238
  name: warbler
211
239
  version_requirements: !ruby/object:Gem::Requirement
@@ -260,22 +288,24 @@ files:
260
288
  - lib/mode/commands/import.rb
261
289
  - lib/mode/commands/login.rb
262
290
  - lib/mode/config.rb
263
- - lib/mode/connector/connect.rb
291
+ - lib/mode/connector/commands/select_report_run_dataset.rb
292
+ - lib/mode/connector/commands/select_table_metadata.rb
264
293
  - lib/mode/connector/daemon.rb
265
294
  - lib/mode/connector/daemonizer.rb
266
- - lib/mode/connector/data_source.rb
295
+ - lib/mode/connector/data_sources/base.rb
296
+ - lib/mode/connector/databases/rdbms.rb
267
297
  - lib/mode/connector/dataset.rb
268
- - lib/mode/connector/message.rb
298
+ - lib/mode/connector/dispatcher.rb
269
299
  - lib/mode/connector/poller.rb
270
- - lib/mode/connector/processor.rb
271
300
  - lib/mode/connector/registrar.rb
272
301
  - lib/mode/connector/scheduler.rb
273
302
  - lib/mode/connector/selector.rb
303
+ - lib/mode/connector/tables/rdbms.rb
274
304
  - lib/mode/connector/type_map.rb
275
- - lib/mode/connector/uploader.rb
276
305
  - lib/mode/logger.rb
277
306
  - lib/mode/version.rb
278
307
  - mode.gemspec
308
+ - script/console.rb
279
309
  - spec/api/form_spec.rb
280
310
  - spec/api/link_spec.rb
281
311
  - spec/api/request_spec.rb
@@ -288,17 +318,19 @@ files:
288
318
  - spec/commands/import_spec.rb
289
319
  - spec/commands/login_spec.rb
290
320
  - spec/config_spec.rb
321
+ - spec/connector/commands/select_report_run_dataset_spec.rb
322
+ - spec/connector/commands/select_table_metadata_spec.rb
291
323
  - spec/connector/daemon_spec.rb
292
324
  - spec/connector/daemonizer_spec.rb
293
- - spec/connector/data_source_spec.rb
294
- - spec/connector/message_spec.rb
325
+ - spec/connector/data_sources/base_spec.rb
326
+ - spec/connector/databases/rdbms_spec.rb
327
+ - spec/connector/dispatcher_spec.rb
295
328
  - spec/connector/poller_spec.rb
296
- - spec/connector/processor_spec.rb
297
329
  - spec/connector/registrar_spec.rb
298
330
  - spec/connector/scheduler_spec.rb
299
331
  - spec/connector/selector_spec.rb
332
+ - spec/connector/tables/rdbms_spec.rb
300
333
  - spec/connector/type_map_spec.rb
301
- - spec/connector/uploader_spec.rb
302
334
  - spec/fixtures/country-codes/README.md
303
335
  - spec/fixtures/country-codes/data/country-codes.csv
304
336
  - spec/fixtures/country-codes/datapackage.json
@@ -347,17 +379,19 @@ test_files:
347
379
  - spec/commands/import_spec.rb
348
380
  - spec/commands/login_spec.rb
349
381
  - spec/config_spec.rb
382
+ - spec/connector/commands/select_report_run_dataset_spec.rb
383
+ - spec/connector/commands/select_table_metadata_spec.rb
350
384
  - spec/connector/daemon_spec.rb
351
385
  - spec/connector/daemonizer_spec.rb
352
- - spec/connector/data_source_spec.rb
353
- - spec/connector/message_spec.rb
386
+ - spec/connector/data_sources/base_spec.rb
387
+ - spec/connector/databases/rdbms_spec.rb
388
+ - spec/connector/dispatcher_spec.rb
354
389
  - spec/connector/poller_spec.rb
355
- - spec/connector/processor_spec.rb
356
390
  - spec/connector/registrar_spec.rb
357
391
  - spec/connector/scheduler_spec.rb
358
392
  - spec/connector/selector_spec.rb
393
+ - spec/connector/tables/rdbms_spec.rb
359
394
  - spec/connector/type_map_spec.rb
360
- - spec/connector/uploader_spec.rb
361
395
  - spec/fixtures/country-codes/README.md
362
396
  - spec/fixtures/country-codes/data/country-codes.csv
363
397
  - spec/fixtures/country-codes/datapackage.json
@@ -1,11 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require 'mode'
4
-
5
- runner = Mode::Connector::Runner.new(:max_jobs => ARGV[0] || 4)
6
-
7
- trap("INT") { runner.stop }
8
- trap("HUP") { runner.stop }
9
- trap("TERM") { runner.stop }
10
-
11
- runner.start
@@ -1,171 +0,0 @@
1
- require 'sequel'
2
-
3
- module Mode
4
- module Connector
5
- class DataSource
6
-
7
- attr_reader :name
8
- attr_reader :props
9
-
10
- attr_reader :connection
11
-
12
- def initialize(name, props = {})
13
- @name = name
14
- @props = props
15
- end
16
-
17
- def adapter
18
- props['adapter']
19
- end
20
-
21
- def username
22
- props['username']
23
- end
24
-
25
- def password
26
- props['password']
27
- end
28
-
29
- def host
30
- props['host']
31
- end
32
-
33
- def port
34
- props['port']
35
- end
36
-
37
- def database
38
- props['database']
39
- end
40
-
41
- def ssl
42
- props['ssl']
43
- end
44
- alias_method :ssl?, :ssl
45
-
46
- def select(query, &block)
47
- log_connection_query(query)
48
- connection.dataset.fetch_rows(query, &block)
49
- end
50
-
51
- def connection
52
- @connection ||= Sequel.connect(connection_url, adapter_opts).tap do |conn|
53
- conn.extension(:connection_validator)
54
- end
55
- end
56
-
57
- def jdbc?
58
- adapter.start_with?('jdbc')
59
- end
60
-
61
- def vertica?
62
- ['vertica', 'jdbc:vertica'].include?(adapter)
63
- end
64
-
65
- def redshift?
66
- ['redshift', 'jdbc:redshift'].include?(adapter)
67
- end
68
-
69
- def sqlserver?
70
- ['tiny_tds', 'jdbc:sqlserver'].include?(adapter)
71
- end
72
-
73
- def postgres?
74
- ['postgres', 'jdbc:postgresql'].include?(adapter)
75
- end
76
-
77
- def mysql?
78
- ['mysql', 'mysql2', 'jdbc:mysql'].include?(adapter)
79
- end
80
-
81
- def oracle?
82
- ['oracle', 'jdbc:oracle:thin'].include?(adapter)
83
- end
84
-
85
- private
86
-
87
- def adapter_opts
88
- opts = {}
89
-
90
- if redshift?
91
- opts.merge!({
92
- :client_min_messages => '',
93
- :force_standard_strings => false
94
- })
95
- elsif postgres?
96
- opts.merge!({
97
- :sslmode => (ssl? ? ssl : 'prefer')
98
- })
99
- end
100
-
101
- opts
102
- end
103
-
104
- def adapter_segment
105
- case adapter
106
- when 'jdbc:redshift'
107
- 'jdbc:postgresql'
108
- # when 'jdbc:oracle:thin'
109
- # jdbc:oracle:thin:scott/tiger@localhost:1521:orcl
110
- else
111
- adapter
112
- end
113
- end
114
-
115
-
116
- def port_segment
117
- port.nil? ? nil : ":#{port}"
118
- end
119
-
120
- def password_segment
121
- jdbc? ? jdbc_password_segment : standard_password_segment
122
- end
123
-
124
- def jdbc_password_segment
125
- password.nil? ? nil : "&password=#{password}"
126
- end
127
-
128
- def standard_password_segment
129
- password.nil? ? nil : ":#{password}"
130
- end
131
-
132
- def ssl_segment
133
- jdbc? ? jdbc_ssl_segment : nil
134
- end
135
-
136
- def jdbc_ssl_segment
137
- if ssl?
138
- if mysql?
139
- "&useSSL=true"
140
- elsif sqlserver?
141
- "&encrypt=true&trustServerCertificate=true"
142
- elsif postgres? || vertica? || redshift?
143
- "&ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory"
144
- else
145
- nil
146
- end
147
- else
148
- nil
149
- end
150
- end
151
-
152
- def connection_url
153
- jdbc? ? jdbc_connection_url : standard_connection_url
154
- end
155
-
156
- def jdbc_connection_url
157
- "#{adapter_segment}://#{host}#{port_segment}/#{database}?user=#{username}#{password_segment}#{ssl_segment}"
158
- end
159
-
160
- def standard_connection_url
161
- "#{adapter_segment}://#{username}#{password_segment}@#{host}#{port_segment}/#{database}"
162
- end
163
-
164
- def log_connection_query(query)
165
- return if query.nil?
166
- Mode::Logger.instance.debug(
167
- "Connect::DataSource", "QUERY", query.split("\n"))
168
- end
169
- end
170
- end
171
- end