mode 0.0.5 → 0.0.7

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -1
  3. data/README.md +17 -22
  4. data/bin/mode +1 -1
  5. data/lib/mode.rb +34 -6
  6. data/lib/mode/api/form.rb +53 -0
  7. data/lib/mode/api/link.rb +31 -0
  8. data/lib/mode/api/request.rb +181 -0
  9. data/lib/mode/api/resource.rb +67 -0
  10. data/lib/mode/auth/access_token.rb +23 -0
  11. data/lib/mode/cli.rb +3 -3
  12. data/lib/mode/cli/analyze.rb +1 -1
  13. data/lib/mode/cli/base.rb +5 -0
  14. data/lib/mode/cli/connect.rb +18 -0
  15. data/lib/mode/cli/helpers.rb +0 -9
  16. data/lib/mode/cli/import.rb +9 -38
  17. data/lib/mode/cli/login.rb +13 -0
  18. data/lib/mode/cli/package.rb +2 -5
  19. data/lib/mode/commands/analyze_field.rb +20 -21
  20. data/lib/mode/commands/analyze_schema.rb +69 -48
  21. data/lib/mode/commands/connect.rb +78 -0
  22. data/lib/mode/commands/helpers.rb +54 -0
  23. data/lib/mode/commands/import.rb +209 -20
  24. data/lib/mode/commands/login.rb +111 -0
  25. data/lib/mode/config.rb +13 -33
  26. data/lib/mode/configurable.rb +46 -0
  27. data/lib/mode/connector/config.rb +31 -0
  28. data/lib/mode/connector/daemon.rb +27 -0
  29. data/lib/mode/connector/data_source.rb +75 -0
  30. data/lib/mode/connector/dataset.rb +13 -0
  31. data/lib/mode/connector/message.rb +31 -0
  32. data/lib/mode/connector/poller.rb +27 -0
  33. data/lib/mode/connector/processor.rb +58 -0
  34. data/lib/mode/connector/registrar.rb +36 -0
  35. data/lib/mode/connector/scheduler.rb +62 -0
  36. data/lib/mode/connector/selector.rb +47 -0
  37. data/lib/mode/connector/type_map.rb +45 -0
  38. data/lib/mode/connector/uploader.rb +50 -0
  39. data/lib/mode/logger.rb +202 -0
  40. data/lib/mode/version.rb +1 -1
  41. data/mode.gemspec +13 -2
  42. data/spec/api/form_spec.rb +51 -0
  43. data/spec/api/link_spec.rb +23 -0
  44. data/spec/api/request_spec.rb +111 -0
  45. data/spec/api/resource_spec.rb +70 -0
  46. data/spec/auth/access_token_spec.rb +22 -0
  47. data/spec/commands/analyze_field_spec.rb +26 -0
  48. data/spec/commands/analyze_schema_spec.rb +7 -5
  49. data/spec/commands/connect_spec.rb +80 -0
  50. data/spec/commands/helpers_spec.rb +69 -0
  51. data/spec/commands/import_spec.rb +155 -0
  52. data/spec/commands/login_spec.rb +178 -0
  53. data/spec/config_spec.rb +9 -7
  54. data/spec/connector/config_spec.rb +46 -0
  55. data/spec/connector/daemon_spec.rb +30 -0
  56. data/spec/connector/data_source_spec.rb +73 -0
  57. data/spec/connector/message_spec.rb +22 -0
  58. data/spec/connector/poller_spec.rb +26 -0
  59. data/spec/connector/processor_spec.rb +93 -0
  60. data/spec/connector/registrar_spec.rb +53 -0
  61. data/spec/connector/scheduler_spec.rb +93 -0
  62. data/spec/connector/selector_spec.rb +54 -0
  63. data/spec/connector/type_map_spec.rb +45 -0
  64. data/spec/connector/uploader_spec.rb +55 -0
  65. data/spec/fixtures/country-codes/README.md +71 -0
  66. data/spec/fixtures/country-codes/data/country-codes.csv +250 -0
  67. data/spec/fixtures/country-codes/datapackage.json +142 -0
  68. data/spec/fixtures/country-codes/scripts/get_countries_of_earth.py +370 -0
  69. data/spec/fixtures/country-codes/scripts/reorder_columns.py +8 -0
  70. data/spec/fixtures/country-codes/scripts/requirements.pip +2 -0
  71. data/spec/fixtures/espn_draft.csv +473 -1
  72. data/spec/fixtures/espn_draft/data.csv +473 -0
  73. data/spec/fixtures/espn_draft/datapackage.json +43 -0
  74. data/spec/logger_spec.rb +79 -0
  75. data/spec/spec_helper.rb +6 -1
  76. metadata +156 -19
  77. data/lib/mode/cli/setup.rb +0 -12
  78. data/lib/mode/commands/package.rb +0 -56
  79. data/lib/mode/commands/setup.rb +0 -36
  80. data/lib/mode/package_builder.rb +0 -57
  81. data/spec/commands/setup_spec.rb +0 -62
  82. data/spec/fixtures/MOCK_DATA.csv +0 -100001
  83. data/spec/fixtures/cb_clean_small.csv +0 -100000
  84. data/spec/fixtures/duplicate_keys.csv +0 -3
  85. data/spec/fixtures/format_examples.csv.txt +0 -6
  86. data/spec/fixtures/format_examples_after_excel.csv.txt +0 -1
@@ -0,0 +1,178 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Commands::Login do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+ let(:username) { 'besquared' }
6
+ let(:password) { 'password' }
7
+
8
+ let(:personal_repr) {
9
+ {
10
+ 'name' => 'personal',
11
+ 'token' => 'token123',
12
+ 'account_name' => 'besquared'
13
+ }
14
+ }
15
+
16
+ let(:secondary_repr) {
17
+ {
18
+ 'name' => 'secondary',
19
+ 'token' => 'token567',
20
+ 'account_name' => 'besquared'
21
+ }
22
+ }
23
+
24
+ let(:tokens_resource) {
25
+ Mode::API::Resource.new({
26
+ '_embedded' => {
27
+ 'access_tokens' => [
28
+ personal_repr,
29
+ secondary_repr
30
+ ]
31
+ }
32
+ })
33
+ }
34
+
35
+ let(:single_tokens_resource) {
36
+ Mode::API::Resource.new({
37
+ '_embedded' => {
38
+ 'access_tokens' => [
39
+ personal_repr
40
+ ]
41
+ }
42
+ })
43
+ }
44
+
45
+ let(:personal_token) {
46
+ Mode::Auth::AccessToken.new(
47
+ Mode::API::Resource.new(personal_repr)
48
+ )
49
+ }
50
+
51
+ let(:secondary_token) {
52
+ Mode::Auth::AccessToken.new(
53
+ Mode::API::Resource.new(secondary_repr)
54
+ )
55
+ }
56
+
57
+ before do
58
+ initialize_logger
59
+ end
60
+
61
+ it "initializes with each environment" do
62
+ [:test, :staging, :development, :production].each do |env|
63
+ Mode::Commands::Login.new(env => true)
64
+ end
65
+ end
66
+
67
+ it "executes a login process with a new config" do
68
+ command = Mode::Commands::Login.new
69
+
70
+ command.stub(:say)
71
+ command.stub(:config_dir).and_return(tmpdir)
72
+
73
+ command.should_receive(:configure_credentials).and_return([username, password])
74
+ command.should_receive(:choose_access_token).and_return(personal_token)
75
+
76
+ command.execute
77
+
78
+ config = Mode::Config.new(tmpdir)
79
+
80
+ config.username.should == 'besquared'
81
+ config.access_token.should == 'token123'
82
+ end
83
+
84
+ it "executes a login process with an existing config" do
85
+ command = Mode::Commands::Login.new
86
+
87
+ command.stub(:say)
88
+ command.stub(:config_dir).and_return(tmpdir)
89
+
90
+ Mode::Config.init(tmpdir)
91
+
92
+ command.should_receive(:configure_credentials).and_return([username, password])
93
+ command.should_receive(:choose_access_token).and_return(personal_token)
94
+
95
+ command.execute
96
+
97
+ config = Mode::Config.new(tmpdir)
98
+
99
+ config.username.should == 'besquared'
100
+ config.access_token.should == 'token123'
101
+ end
102
+
103
+ it "rescues and displays errors during the login process" do
104
+ command = Mode::Commands::Login.new
105
+
106
+ command.stub(:config_dir).and_return(tmpdir)
107
+
108
+ error = StandardError.new("Expected Break")
109
+ command.should_receive(:configure_credentials).and_raise(error)
110
+
111
+ command.should_receive(:say).with("Expected Break")
112
+
113
+ command.execute
114
+ end
115
+
116
+ it "configures API with credentials" do
117
+ command = Mode::Commands::Login.new(:test => true)
118
+
119
+ command.send(:configure_api, 'besquared', 'password')
120
+
121
+ Mode::API::Request.environment.should == :test
122
+ end
123
+
124
+ it "requests credentials and configures API" do
125
+ command = Mode::Commands::Login.new(:test => true)
126
+
127
+ command.should_receive(:configure_api)
128
+ command.should_receive(:ask).and_return('besquared', 'password')
129
+
130
+ username, password = command.send(:configure_credentials)
131
+
132
+ username.should == 'besquared'
133
+ password.should == 'password'
134
+ end
135
+
136
+ it "raises when credentials are invalid" do
137
+ command = Mode::Commands::Login.new(:test => true)
138
+
139
+ response = double(:response, :status => 401)
140
+ Mode::API::Request.should_receive(:get).with(:access_tokens).and_return(response)
141
+ expect { command.send(:fetch_access_tokens) }.to raise_error
142
+ end
143
+
144
+ it "fetches access tokens from the API" do
145
+ command = Mode::Commands::Login.new(:test => true)
146
+ Mode::API::Request.should_receive(:get).with(:access_tokens).and_return(tokens_resource)
147
+
148
+ tokens = command.send(:fetch_access_tokens)
149
+
150
+ tokens.length.should == 2
151
+ tokens.first.token.should == 'token123'
152
+ tokens.last.token.should == 'token567'
153
+ end
154
+
155
+ it "chooses the first token if there is only one" do
156
+ command = Mode::Commands::Login.new(:test => true)
157
+ Mode::API::Request.should_receive(:get).with(:access_tokens).and_return(single_tokens_resource)
158
+
159
+ command.should_not_receive(:ask)
160
+
161
+ chosen_token = command.send(:choose_access_token)
162
+
163
+ chosen_token.should_not == nil
164
+ chosen_token.token.should == 'token123'
165
+ end
166
+
167
+ it "prompts the user to choose a token" do
168
+ command = Mode::Commands::Login.new(:test => true)
169
+ Mode::API::Request.should_receive(:get).with(:access_tokens).and_return(tokens_resource)
170
+
171
+ command.should_receive(:ask).and_return("2")
172
+
173
+ chosen_token = command.send(:choose_access_token)
174
+
175
+ chosen_token.should_not == nil
176
+ chosen_token.token.should == 'token567'
177
+ end
178
+ end
@@ -1,12 +1,14 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe Mode::Config do
4
- before(:each) do
5
- @tmpdir = Dir.mktmpdir
6
- end
4
+ let(:tmpdir) { Dir.mktmpdir }
7
5
 
8
6
  it "should initialize" do
9
- config = Mode::Config.init(@tmpdir)
7
+ Mode::Config.exists?(tmpdir).should == false
8
+
9
+ config = Mode::Config.init(tmpdir)
10
+ Mode::Config.exists?(tmpdir).should == true
11
+
10
12
  config.username.should == nil
11
13
  config.access_token.should == nil
12
14
  end
@@ -16,7 +18,7 @@ describe Mode::Config do
16
18
  end
17
19
 
18
20
  it "should save" do
19
- config = Mode::Config.init(@tmpdir)
21
+ config = Mode::Config.init(tmpdir)
20
22
  config.username = 'besquared'
21
23
  config.access_token = 'token'
22
24
  config.save
@@ -25,10 +27,10 @@ describe Mode::Config do
25
27
  end
26
28
 
27
29
  it "should have FILENAME" do
28
- Mode::Config::FILENAME.should == '.mode.yml'
30
+ Mode::Config::default_filename.should == 'config.yml'
29
31
  end
30
32
 
31
33
  it "should generate full path" do
32
- Mode::Config.full_path(@tmpdir).should == File.join(@tmpdir, '.mode.yml')
34
+ Mode::Config.full_path(tmpdir).should == File.join(tmpdir, 'config.yml')
33
35
  end
34
36
  end
@@ -0,0 +1,46 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Config do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+
6
+ before do
7
+ initialize_logger
8
+ end
9
+
10
+ it "should initialize" do
11
+ Mode::Connector::Config.exists?(tmpdir).should == false
12
+ config = Mode::Connector::Config.init(tmpdir)
13
+ Mode::Connector::Config.exists?(tmpdir).should == true
14
+
15
+ config.data_sources.should == []
16
+ end
17
+
18
+ it "should initialize with data sources" do
19
+ config = Mode::Connector::Config.init(tmpdir)
20
+ config.data_sources << Mode::Connector::DataSource.new('test', {})
21
+ config.save
22
+
23
+ config = Mode::Connector::Config.new(tmpdir)
24
+ config.data_sources.length.should == 1
25
+ end
26
+
27
+ it "should raise if config file doesn't exist" do
28
+ expect { config = Mode::Connector::Config.new('fake.yml') }.to raise_error
29
+ end
30
+
31
+ it "should save" do
32
+ config = Mode::Connector::Config.init(tmpdir)
33
+ config.data_sources << Mode::Connector::DataSource.new('test', {})
34
+ config.save
35
+
36
+ File.read(config.path).should == config.send(:to_yaml)
37
+ end
38
+
39
+ it "should have FILENAME" do
40
+ Mode::Connector::Config::default_filename.should == 'connect.yml'
41
+ end
42
+
43
+ it "should generate full path" do
44
+ Mode::Connector::Config.full_path(tmpdir).should == File.join(tmpdir, 'connect.yml')
45
+ end
46
+ end
@@ -0,0 +1,30 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Daemon do
4
+ let(:working_dir) { Dir.mktmpdir }
5
+ let(:scheduler) { double(:scheduler, :start! => true, :stop! => true) }
6
+
7
+ before do
8
+ initialize_logger
9
+ end
10
+
11
+ it "starts a daemon" do
12
+ daemon = Mode::Connector::Daemon.new(:working_dir => working_dir)
13
+ Mode::Connector::Scheduler.should_receive(:new).and_return(scheduler)
14
+
15
+ begin
16
+ daemon.start([5, []])
17
+ ensure
18
+ daemon.stop
19
+ end
20
+ end
21
+
22
+ it "logs errors on failure" do
23
+ daemon = Mode::Connector::Daemon.new(:working_dir => working_dir)
24
+ Mode::Connector::Scheduler.should_receive(:new).and_raise(StandardError.new("BREAK"))
25
+
26
+ Mode::Logger.instance.should_receive(:error)
27
+
28
+ daemon.start([5, []])
29
+ end
30
+ end
@@ -0,0 +1,73 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::DataSource do
4
+ let(:props) {
5
+ {
6
+ 'adapter' => 'postgres',
7
+ 'host' => 'localhost',
8
+ 'username' => 'postgres',
9
+ 'database' => 'warehouse',
10
+ 'password' => nil
11
+ }
12
+ }
13
+
14
+ let(:redprops) {
15
+ {
16
+ 'adapter' => 'redshift',
17
+ 'host' => 'localhost',
18
+ 'username' => 'postgres',
19
+ 'database' => 'warehouse',
20
+ 'password' => nil
21
+ }
22
+ }
23
+
24
+ it "initializes with props" do
25
+ source = Mode::Connector::DataSource.new('dev', props)
26
+ source.adapter.should == 'postgres'
27
+ source.host.should == 'localhost'
28
+ source.username.should == 'postgres'
29
+ source.database.should == 'warehouse'
30
+ source.password.should == nil
31
+ end
32
+
33
+ before do
34
+ initialize_logger
35
+ end
36
+
37
+ it "has a connection url" do
38
+ source = Mode::Connector::DataSource.new('dev', props)
39
+ source.send(:connection_url).should == "postgres://postgres@localhost/warehouse"
40
+ end
41
+
42
+ it "has a connection" do
43
+ source = Mode::Connector::DataSource.new('dev', props)
44
+ Sequel.should_receive(:connect).and_return(:connect)
45
+ source.connection.should == :connect
46
+ end
47
+
48
+ it 'has a connection for redshift' do
49
+ source = Mode::Connector::DataSource.new('dev', redprops)
50
+ Sequel.should_receive(:connect).and_return(:connect)
51
+ source.connection.should == :connect
52
+ end
53
+
54
+ it 'has a connection dataset' do
55
+ source = Mode::Connector::DataSource.new('dev', props)
56
+
57
+ dataset = source.send(:connection_dataset, "SELECT * FROM fake")
58
+
59
+ dataset.should_not == nil
60
+ end
61
+
62
+ it "fetches results from dataset" do
63
+ source = Mode::Connector::DataSource.new('dev', props)
64
+
65
+ dataset = double(:dataset)
66
+ dataset.should_receive(:each).and_yield({'cnt' => 5})
67
+ source.should_receive(:connection_dataset).with("SELECT 1").and_return(dataset)
68
+
69
+ source.select("SELECT 1") do |row|
70
+ row['cnt'].should == 5
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Message do
4
+ let(:resource) {
5
+ Mode::API::Resource.new({
6
+ 'type' => 'query',
7
+ 'name' => 'fake',
8
+ 'query' => 'query'
9
+ })
10
+ }
11
+
12
+ before do
13
+ initialize_logger
14
+ end
15
+
16
+ it "initializes with resource" do
17
+ message = Mode::Connector::Message.new(resource)
18
+ message.type.should == 'query'
19
+ message.name.should == 'fake'
20
+ message.query.should == 'query'
21
+ end
22
+ end
@@ -0,0 +1,26 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Poller do
4
+ let(:path) {
5
+ Mode::API::Request.data_source_connection_messages_path
6
+ }
7
+
8
+ before do
9
+ initialize_logger
10
+ end
11
+
12
+ it "initializes and performs poll" do
13
+ poller = Mode::Connector::Poller.new
14
+
15
+ resource = Mode::API::Resource.new({
16
+ '_embedded' => {
17
+ 'messages' => [{}, {}]
18
+ }
19
+ })
20
+ Mode::API::Request.should_receive(:get).with(path).and_return(resource)
21
+
22
+ poller.perform! do |message|
23
+ message.should be_instance_of(Mode::Connector::Message)
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,93 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Processor do
4
+ let(:data_sources) {
5
+ [
6
+ Mode::Connector::DataSource.new('testdb', {
7
+ 'adapter' => 'postgres',
8
+ 'host' => 'localhost',
9
+ 'username' => 'postgres',
10
+ 'database' => 'warehouse',
11
+ 'password' => nil
12
+ })
13
+ ]
14
+ }
15
+
16
+ let(:execution_path) { '/execution' }
17
+
18
+ let(:resource) {
19
+ Mode::API::Resource.new({
20
+ 'type' => 'query',
21
+ 'query' => 'SELECT 1',
22
+ '_embedded' => {
23
+ 'data_source' => {
24
+ 'name' => 'testdb'
25
+ }
26
+ },
27
+ '_links' => {
28
+ 'execution' => {
29
+ 'href' => execution_path
30
+ }
31
+ }
32
+ })
33
+ }
34
+
35
+ let(:message) {
36
+ Mode::Connector::Message.new(resource)
37
+ }
38
+
39
+ before do
40
+ initialize_logger
41
+ end
42
+
43
+ it "initializes with message and data sources" do
44
+ processor = Mode::Connector::Processor.new(message, data_sources)
45
+
46
+ processor.message.should_not == nil
47
+ processor.data_sources.should_not == nil
48
+ end
49
+
50
+ it "selects a dataset" do
51
+ processor = Mode::Connector::Processor.new(message, data_sources)
52
+
53
+ selector = double(:selector, :perform! => true)
54
+ Mode::Connector::Selector.should_receive(:new).with(
55
+ message.query, data_sources.first).and_return(selector)
56
+
57
+ processor.send(:select!)
58
+ end
59
+
60
+ it "sends a dataset" do
61
+ processor = Mode::Connector::Processor.new(message, data_sources)
62
+
63
+ dataset = double(:dataset)
64
+ uploader = double(:uploader, :perform! => true)
65
+ Mode::Connector::Uploader.should_receive(:new).with(execution_path, dataset).and_return(uploader)
66
+
67
+ processor.send(:send!, dataset)
68
+ end
69
+
70
+ it "sends an error" do
71
+ processor = Mode::Connector::Processor.new(message, data_sources)
72
+
73
+ Mode::Connector::Uploader.should_receive(:error!).with(
74
+ execution_path, "message", "details").and_return(true)
75
+
76
+ processor.send(:error!, "message", "details")
77
+ end
78
+
79
+ it "performs a send" do
80
+ processor = Mode::Connector::Processor.new(message, data_sources)
81
+ processor.should_receive(:select!).and_return(true)
82
+ processor.should_receive(:send!).and_return(true)
83
+ processor.perform!
84
+ end
85
+
86
+ it "performs an error" do
87
+ processor = Mode::Connector::Processor.new(message, data_sources)
88
+ processor.should_receive(:select!).and_raise(StandardError.new("BREAK!"))
89
+ processor.should_receive(:error!).and_return(true)
90
+
91
+ processor.perform!
92
+ end
93
+ end