mode 0.0.17 → 0.0.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -134
  3. data/lib/connect.rb +5 -5
  4. data/lib/mode.rb +12 -8
  5. data/lib/mode/api/form.rb +30 -8
  6. data/lib/mode/api/request.rb +34 -16
  7. data/lib/mode/commands/connect.rb +0 -8
  8. data/lib/mode/commands/import.rb +1 -1
  9. data/lib/mode/config.rb +9 -1
  10. data/lib/mode/connector/commands/select_report_run_dataset.rb +84 -0
  11. data/lib/mode/connector/commands/select_table_metadata.rb +113 -0
  12. data/lib/mode/connector/daemon.rb +3 -3
  13. data/lib/mode/connector/data_sources/base.rb +191 -0
  14. data/lib/mode/connector/databases/rdbms.rb +69 -0
  15. data/lib/mode/connector/dataset.rb +0 -1
  16. data/lib/mode/connector/dispatcher.rb +27 -0
  17. data/lib/mode/connector/poller.rb +2 -3
  18. data/lib/mode/connector/registrar.rb +74 -16
  19. data/lib/mode/connector/scheduler.rb +24 -15
  20. data/lib/mode/connector/selector.rb +1 -0
  21. data/lib/mode/connector/tables/rdbms.rb +130 -0
  22. data/lib/mode/version.rb +2 -2
  23. data/mode.gemspec +2 -0
  24. data/script/console.rb +11 -0
  25. data/spec/api/form_spec.rb +17 -8
  26. data/spec/api/request_spec.rb +3 -3
  27. data/spec/commands/connect_spec.rb +1 -10
  28. data/spec/config_spec.rb +1 -1
  29. data/spec/connector/commands/select_report_run_dataset_spec.rb +96 -0
  30. data/spec/connector/commands/select_table_metadata_spec.rb +115 -0
  31. data/spec/connector/{data_source_spec.rb → data_sources/base_spec.rb} +8 -8
  32. data/spec/connector/databases/rdbms_spec.rb +43 -0
  33. data/spec/connector/dispatcher_spec.rb +47 -0
  34. data/spec/connector/poller_spec.rb +2 -1
  35. data/spec/connector/registrar_spec.rb +111 -26
  36. data/spec/connector/scheduler_spec.rb +12 -31
  37. data/spec/connector/selector_spec.rb +1 -1
  38. data/spec/connector/tables/rdbms_spec.rb +85 -0
  39. metadata +49 -15
  40. data/lib/mode/connector/connect.rb +0 -11
  41. data/lib/mode/connector/data_source.rb +0 -171
  42. data/lib/mode/connector/message.rb +0 -31
  43. data/lib/mode/connector/processor.rb +0 -58
  44. data/lib/mode/connector/uploader.rb +0 -54
  45. data/spec/connector/message_spec.rb +0 -22
  46. data/spec/connector/processor_spec.rb +0 -93
  47. data/spec/connector/uploader_spec.rb +0 -57
@@ -15,7 +15,7 @@ describe Mode::API::Request do
15
15
  }
16
16
  })
17
17
  end
18
-
18
+
19
19
  before do
20
20
  initialize_logger
21
21
  end
@@ -46,7 +46,7 @@ describe Mode::API::Request do
46
46
  'test' => 'test',
47
47
  'development' => 'localhost',
48
48
  'staging' => 'staging.modeanalytics.com',
49
- 'production' => 'stealth.modeanalytics.com'
49
+ 'production' => 'modeanalytics.com'
50
50
  }
51
51
 
52
52
  hosts.each do |env, host|
@@ -98,7 +98,7 @@ describe Mode::API::Request do
98
98
  end
99
99
 
100
100
  it 'return log error and return raw response on request errors' do
101
- response = double(:get, {
101
+ response = double(:get, {
102
102
  :status => 500,
103
103
  :success? => false,
104
104
  :body => '{"error": "failed"}'
@@ -6,7 +6,7 @@ describe Mode::Commands::Connect do
6
6
  let(:daemon) { double(:daemon) }
7
7
 
8
8
  let(:data_source) {
9
- Mode::Connector::DataSource.new('testdb', {
9
+ Mode::Connector::DataSources::Base.new('testdb', {
10
10
  'adapter' => 'postgres',
11
11
  'host' => 'localhost',
12
12
  'username' => 'postgres',
@@ -26,14 +26,6 @@ describe Mode::Commands::Connect do
26
26
  connect.command.should == 'start'
27
27
  connect.concurrency.should == 4
28
28
  end
29
-
30
- it "registers connector" do
31
- connect.should_receive(:configuration).and_return(config)
32
-
33
- registrar = double(:registrar, :perform! => true)
34
- Mode::Connector::Registrar.should_receive(:new).and_return(registrar)
35
- connect.send(:register!)
36
- end
37
29
  end
38
30
 
39
31
  describe "controlling the daemon" do
@@ -71,7 +63,6 @@ describe Mode::Commands::Connect do
71
63
  it 'executes the connector' do
72
64
  connect = Mode::Commands::Connect.new('start')
73
65
 
74
- connect.should_receive(:register!)
75
66
  connect.should_receive(:run_command!)
76
67
  connect.should_receive(:validate_config!)
77
68
 
@@ -28,7 +28,7 @@ describe Mode::Config do
28
28
 
29
29
  it "should initialize with data sources" do
30
30
  config = Mode::Config.init(tmpdir)
31
- config.data_sources << Mode::Connector::DataSource.new('test', {})
31
+ config.data_sources << Mode::Connector::DataSources::Base.new('test', {})
32
32
  config.save
33
33
 
34
34
  config = Mode::Config.new(tmpdir)
@@ -0,0 +1,96 @@
1
+ require 'tempfile'
2
+ require 'spec_helper'
3
+
4
+ describe Mode::Connector::Commands::SelectReportRunDataset do
5
+ let(:data_sources) {
6
+ [
7
+ Mode::Connector::DataSources::Base.new('testdb', {
8
+ 'adapter' => 'postgres',
9
+ 'host' => 'localhost',
10
+ 'username' => 'postgres',
11
+ 'database' => 'warehouse',
12
+ 'password' => nil
13
+ })
14
+ ]
15
+ }
16
+
17
+ let(:command) {
18
+ Mode::API::Resource.new({
19
+ 'type' => 'query',
20
+ 'query' => 'SELECT 1',
21
+ '_embedded' => {
22
+ 'data_source' => {
23
+ 'name' => 'testdb'
24
+ }
25
+ },
26
+ '_links' => {
27
+ 'execution' => {
28
+ 'href' => '/reports/5/runs/10'
29
+ }
30
+ }
31
+ })
32
+ }
33
+
34
+ before do
35
+ initialize_logger
36
+ end
37
+
38
+ it "initializes with command and data sources" do
39
+ processor = Mode::Connector::Commands::SelectReportRunDataset.new(command, data_sources)
40
+
41
+ processor.command.should_not == nil
42
+ processor.data_sources.should_not == nil
43
+ end
44
+
45
+ it "selects a dataset" do
46
+ processor = Mode::Connector::Commands::SelectReportRunDataset.new(command, data_sources)
47
+
48
+ selector = double(:selector, :perform! => true)
49
+ Mode::Connector::Selector.should_receive(:new).with(
50
+ command.query, data_sources.first).and_return(selector)
51
+
52
+ processor.send(:dataset)
53
+ end
54
+
55
+ it "uploads a dataset" do
56
+ processor = Mode::Connector::Commands::SelectReportRunDataset.new(command, data_sources)
57
+
58
+ form = double(:form)
59
+ form.should_receive(:submit!).and_return(true)
60
+ command.should_receive(:forms).and_return(form)
61
+
62
+ file = Tempfile.new('foo')
63
+
64
+ dataset = double(:dataset,
65
+ :path => file.path,
66
+ :count => 1,
67
+ :column_types => [{:field => :type}])
68
+
69
+ processor.send(:upload, dataset)
70
+ end
71
+
72
+ it "sends an error" do
73
+ processor = Mode::Connector::Commands::SelectReportRunDataset.new(command, data_sources)
74
+
75
+ form = double(:form)
76
+ form.should_receive(:submit!).and_return(true)
77
+ command.should_receive(:forms).and_return(form)
78
+
79
+ processor.send(:error, "command", "details")
80
+ end
81
+
82
+ it "performs a send" do
83
+ processor = Mode::Connector::Commands::SelectReportRunDataset.new(command, data_sources)
84
+ processor.should_receive(:dataset).and_return(true)
85
+ processor.should_receive(:upload).and_return(true)
86
+ processor.perform!
87
+ end
88
+
89
+ it "performs an error" do
90
+ processor = Mode::Connector::Commands::SelectReportRunDataset.new(command, data_sources)
91
+ processor.should_receive(:dataset).and_raise(StandardError.new("BREAK!"))
92
+ processor.should_receive(:error).and_return(true)
93
+
94
+ processor.perform!
95
+ end
96
+ end
@@ -0,0 +1,115 @@
1
+ require 'tempfile'
2
+ require 'spec_helper'
3
+
4
+ describe Mode::Connector::Commands::SelectTableMetadata do
5
+ let(:data_sources) {
6
+ [
7
+ Mode::Connector::DataSources::Base.new('testdb', {
8
+ 'adapter' => 'postgres',
9
+ 'host' => 'localhost',
10
+ 'username' => 'postgres',
11
+ 'database' => 'warehouse',
12
+ 'password' => nil
13
+ })
14
+ ]
15
+ }
16
+
17
+ let(:command) {
18
+ Mode::API::Resource.new({
19
+ 'command' => {
20
+ 'name' => 'select:table:metadata'
21
+ },
22
+
23
+ '_embedded' => {
24
+ 'data_source' => {
25
+ 'name' => 'testdb'
26
+ },
27
+
28
+ 'table' => {
29
+ 'name' => 'testdb',
30
+ 'schema' => 'public',
31
+
32
+ '_forms' => {
33
+ 'method' => 'put',
34
+ 'action' => 'tables/5',
35
+
36
+ 'edit' => {
37
+ 'input' => {
38
+ # should I test this here?
39
+ }
40
+ }
41
+ }
42
+ }
43
+ },
44
+
45
+ '_forms' => {
46
+ 'edit' => {
47
+ 'method' => 'put',
48
+ 'action' => '/executions/5',
49
+
50
+ 'input' => {
51
+ 'execution' => {
52
+ 'error' => {
53
+ 'message' => {
54
+ 'type' => 'text'
55
+ },
56
+ 'detail' => {
57
+ 'type' => 'text'
58
+ }
59
+ }
60
+ }
61
+ }
62
+ },
63
+
64
+ 'succeed' => {
65
+ 'method' => 'put',
66
+ 'action' => '/executions/5'
67
+ }
68
+ }
69
+ })
70
+ }
71
+
72
+ before do
73
+ initialize_logger
74
+ end
75
+
76
+ it "initializes with command and data sources" do
77
+ commander = Mode::Connector::Commands::SelectTableMetadata.new(command, data_sources)
78
+
79
+ commander.command.should_not == nil
80
+ commander.data_sources.should_not == nil
81
+
82
+ # accessors
83
+ commander.send(:table).should_not == nil
84
+ end
85
+
86
+ it "PUTs table metadata" do
87
+ commander = Mode::Connector::Commands::SelectTableMetadata.new(command, data_sources)
88
+
89
+ file = Tempfile.new('foo')
90
+
91
+ table = double(:table,
92
+ :columns => :columns,
93
+ :row_count => :row_count,
94
+ :preview => double(:preview,
95
+ :count => 10,
96
+ :path => file.path,
97
+ :column_types => {'field' => 'string'},
98
+ )
99
+ )
100
+
101
+ commander.should_receive(:table).and_return(table)
102
+
103
+ Mode::API::Request.should_receive(:put).and_return(true)
104
+
105
+ commander.perform!
106
+ end
107
+
108
+ it "PUTs success on completion" do
109
+ commander = Mode::Connector::Commands::SelectTableMetadata.new(command, data_sources)
110
+
111
+ Mode::API::Request.should_receive(:put).and_return(:success)
112
+
113
+ commander.send(:succeed).should == :success
114
+ end
115
+ end
@@ -1,6 +1,6 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe Mode::Connector::DataSource do
3
+ describe Mode::Connector::DataSources::Base do
4
4
  let(:props) {
5
5
  {
6
6
  'adapter' => 'postgres',
@@ -24,11 +24,11 @@ describe Mode::Connector::DataSource do
24
24
  let(:conn) { double(:conn, :extension => true) }
25
25
 
26
26
  it "initializes with props" do
27
- source = Mode::Connector::DataSource.new('dev', props)
27
+ source = Mode::Connector::DataSources::Base.new('dev', props)
28
28
  source.adapter.should == 'postgres'
29
29
  source.host.should == 'localhost'
30
30
  source.username.should == 'postgres'
31
- source.database.should == 'warehouse'
31
+ source.database_name.should == 'warehouse'
32
32
  source.password.should == nil
33
33
  end
34
34
 
@@ -37,29 +37,29 @@ describe Mode::Connector::DataSource do
37
37
  end
38
38
 
39
39
  it "has a connection url" do
40
- source = Mode::Connector::DataSource.new('dev', props)
40
+ source = Mode::Connector::DataSources::Base.new('dev', props)
41
41
  source.send(:connection_url).should == "postgres://postgres@localhost/warehouse"
42
42
  end
43
43
 
44
44
  it 'has a jdbc connection url' do
45
- source = Mode::Connector::DataSource.new('dev', props.merge('adapter' => 'jdbc:redshift'))
45
+ source = Mode::Connector::DataSources::Base.new('dev', props.merge('adapter' => 'jdbc:redshift'))
46
46
  source.send(:connection_url).should == "jdbc:postgresql://localhost/warehouse?user=postgres"
47
47
  end
48
48
 
49
49
  it "has a connection" do
50
- source = Mode::Connector::DataSource.new('dev', props)
50
+ source = Mode::Connector::DataSources::Base.new('dev', props)
51
51
  Sequel.should_receive(:connect).and_return(conn)
52
52
  source.connection.should == conn
53
53
  end
54
54
 
55
55
  it 'has a connection for redshift' do
56
- source = Mode::Connector::DataSource.new('dev', redprops)
56
+ source = Mode::Connector::DataSources::Base.new('dev', redprops)
57
57
  Sequel.should_receive(:connect).and_return(conn)
58
58
  source.connection.should == conn
59
59
  end
60
60
 
61
61
  it "fetches results from dataset" do
62
- source = Mode::Connector::DataSource.new('dev', props)
62
+ source = Mode::Connector::DataSources::Base.new('dev', props)
63
63
 
64
64
  dataset = double(:dataset)
65
65
  connection = double(:conn, :dataset => dataset)
@@ -0,0 +1,43 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Tables::RDBMS do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+ let(:config) { Mode::Config.init(tmpdir) }
6
+
7
+ let(:data_source) {
8
+ Mode::Connector::DataSources::Base.new('testdb', {
9
+ 'adapter' => 'postgres',
10
+ 'host' => 'localhost',
11
+ 'username' => 'postgres',
12
+ 'database' => 'warehouse',
13
+ 'password' => nil
14
+ })
15
+ }
16
+
17
+ before do
18
+ initialize_logger
19
+ end
20
+
21
+ it "initializes" do
22
+ database = Mode::Connector::Databases::RDBMS.new(data_source, 'warehouse')
23
+
24
+ database.data_source.should == data_source
25
+ database.database_name.should == 'warehouse'
26
+ end
27
+
28
+ it "queries and returns columns" do
29
+ column = { :table_name => 'atable' }
30
+ data_source.should_receive(:select).and_yield(column)
31
+
32
+ database = Mode::Connector::Databases::RDBMS.new(data_source, 'warehouse')
33
+
34
+ database.columns.should == [column]
35
+ end
36
+
37
+ it "returns a table object" do
38
+ Mode::Connector::Tables::RDBMS.should_receive(:new).and_return(:table)
39
+ database = Mode::Connector::Databases::RDBMS.new(data_source, 'warehouse')
40
+
41
+ database.tables('public', 'accounts').should == :table
42
+ end
43
+ end
@@ -0,0 +1,47 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Dispatcher do
4
+ let(:data_sources) { double(:data_sources) }
5
+
6
+ let(:select_table_metadata) {
7
+ Mode::API::Resource.new({
8
+ 'command' => {
9
+ 'name' => 'select:table:metadata'
10
+ }
11
+ })
12
+ }
13
+
14
+ let(:select_report_run_dataset) {
15
+ Mode::API::Resource.new({
16
+ 'command' => {
17
+ 'name' => 'select:report_run:dataset'
18
+ }
19
+ })
20
+ }
21
+
22
+ before do
23
+ initialize_logger
24
+ end
25
+
26
+ it "initializes" do
27
+ dispatcher = Mode::Connector::Dispatcher.new(select_table_metadata, data_sources)
28
+ end
29
+
30
+ it "dispatches a select report run dataset command" do
31
+ dispatcher = Mode::Connector::Dispatcher.new(select_report_run_dataset, data_sources)
32
+
33
+ commando = double(:commando, :perform! => true)
34
+ Mode::Connector::Commands::SelectReportRunDataset.should_receive(:new).and_return(commando)
35
+
36
+ dispatcher.perform!
37
+ end
38
+
39
+ it "dispatches a select table metadata command" do
40
+ dispatcher = Mode::Connector::Dispatcher.new(select_table_metadata, data_sources)
41
+
42
+ commando = double(:commando, :perform! => true)
43
+ Mode::Connector::Commands::SelectTableMetadata.should_receive(:new).and_return(commando)
44
+
45
+ dispatcher.perform!
46
+ end
47
+ end
@@ -17,11 +17,12 @@ describe Mode::Connector::Poller do
17
17
  'messages' => [{}, {}]
18
18
  }
19
19
  })
20
+
20
21
  Mode::API::Request.should_receive(:get).with(path,
21
22
  {:num_messages => 5}).and_return(resource)
22
23
 
23
24
  poller.perform!(:num_messages => 5) do |message|
24
- message.should be_instance_of(Mode::Connector::Message)
25
+ message.should be_instance_of(Mode::API::Resource)
25
26
  end
26
27
  end
27
28
  end
@@ -1,51 +1,136 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe Mode::Connector::Registrar do
4
- let(:tmpdir) { Dir.mktmpdir }
5
4
 
6
- let(:username) { 'besquared' }
7
- let(:password) { 'token_password '}
8
- let(:config) { Mode::Config.init(tmpdir) }
9
- let(:path) { Mode::API::Request.data_source_connection_path }
10
-
11
- let(:props) {
5
+ let(:edit_form) {
12
6
  {
13
- 'adapter' => 'postgres',
14
- 'host' => 'localhost',
15
- 'username' => 'postgres',
16
- 'database' => 'warehouse',
17
- 'password' => nil
7
+ 'method' => 'put',
8
+ 'action' => '/path',
9
+
10
+ 'input' => {
11
+ 'data_source' => {
12
+ 'tables[]' => {
13
+ 'name' => {
14
+ 'type' => 'text'
15
+ },
16
+
17
+ 'schema' => {
18
+ 'type' => 'text'
19
+ },
20
+
21
+ 'columns[]' => {
22
+ 'name' => {
23
+ 'type' => 'text'
24
+ },
25
+
26
+ 'data_type' => {
27
+ 'type' => 'text'
28
+ },
29
+
30
+ 'is_nullable' => {
31
+ 'type' => 'select',
32
+ 'options' => [true, false],
33
+ 'value' => true
34
+ },
35
+
36
+ 'primary_key' => {
37
+ 'type' => 'select',
38
+ 'options' => [true, false],
39
+ 'value' => false
40
+ }
41
+ }
42
+ }
43
+ }
44
+ }
18
45
  }
19
46
  }
20
47
 
21
- before do
22
- initialize_logger
48
+ let(:connection) {
49
+ Mode::API::Resource.new({
50
+ '_embedded' => {
51
+ 'data_sources' => [{
52
+ 'name' => 'double',
23
53
 
24
- Mode::API::Request.configure('test', {
25
- 'credentials' => {
26
- 'username' => username,
27
- 'password' => password
54
+ '_forms' => {
55
+ 'edit' => edit_form
56
+ }
57
+ }]
28
58
  }
29
59
  })
60
+ }
61
+
62
+ let(:database) {
63
+ double(:warehouse,
64
+ :columns => [{
65
+ :table_schema => 'public',
66
+ :table_name => 'accounts',
67
+ :column_name => 'account_id',
68
+ :data_type => 'character varying',
69
+ :is_nullable => false,
70
+ :primary_key => true
71
+ }]
72
+ )
73
+ }
30
74
 
31
- config.data_sources << Mode::Connector::DataSource.new('dev', props)
75
+ let(:data_sources) {
76
+ [double(:data_source,
77
+ :name => 'double',
78
+ :adapter => 'jdbc:postgresql',
79
+ :host => 'localhost',
80
+ :username => 'postgres',
81
+ :database_name => 'warehouse',
82
+ :database => database,
83
+ :connect => true,
84
+ :extension => true,
85
+ :has_information_schema? => true)]
86
+ }
87
+
88
+ before do
89
+ initialize_logger
32
90
  end
33
91
 
34
92
  it "initializes with config" do
35
- registration = Mode::Connector::Registrar.new(config)
93
+ registration = Mode::Connector::Registrar.new(data_sources)
36
94
  end
37
95
 
38
- it 'performs registration request' do
39
- registration = Mode::Connector::Registrar.new(config)
96
+ it 'should register a connection' do
97
+ registration = Mode::Connector::Registrar.new(data_sources)
40
98
 
41
- Mode::API::Request.should_receive(:put).with(path, {
99
+ Mode::API::Request.should_receive(:put).with(
100
+ :data_source_connection, {
42
101
  :data_source_connection => {
43
102
  :data_sources => [{
44
- :name => "dev",
45
- :adapter => "postgres"
103
+ :name => "double",
104
+ :adapter => "jdbc:postgresql"
46
105
  }]
47
106
  }
48
- })
107
+ }).and_return(connection)
108
+
109
+ registration.should_receive(:register_data_sources).and_return(true)
110
+
111
+ registration.perform!
112
+ end
113
+
114
+ it 'should register connection data sources' do
115
+ registration = Mode::Connector::Registrar.new(data_sources)
116
+
117
+ registration.should_receive(:register_connection).and_return(connection)
118
+
119
+ Mode::API::Request.should_receive(:put).with(
120
+ '/path', {
121
+ 'data_source' => {
122
+ 'tables' => [{
123
+ 'schema' => 'public',
124
+ 'name' => 'accounts',
125
+ 'columns' => [{
126
+ 'name' => 'account_id',
127
+ 'data_type' => 'character varying',
128
+ 'is_nullable' => false,
129
+ 'primary_key' => true
130
+ }]
131
+ }]
132
+ }
133
+ }, {}).and_return(true)
49
134
 
50
135
  registration.perform!
51
136
  end