mode 0.0.5 → 0.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -1
  3. data/README.md +17 -22
  4. data/bin/mode +1 -1
  5. data/lib/mode.rb +34 -6
  6. data/lib/mode/api/form.rb +53 -0
  7. data/lib/mode/api/link.rb +31 -0
  8. data/lib/mode/api/request.rb +181 -0
  9. data/lib/mode/api/resource.rb +67 -0
  10. data/lib/mode/auth/access_token.rb +23 -0
  11. data/lib/mode/cli.rb +3 -3
  12. data/lib/mode/cli/analyze.rb +1 -1
  13. data/lib/mode/cli/base.rb +5 -0
  14. data/lib/mode/cli/connect.rb +18 -0
  15. data/lib/mode/cli/helpers.rb +0 -9
  16. data/lib/mode/cli/import.rb +9 -38
  17. data/lib/mode/cli/login.rb +13 -0
  18. data/lib/mode/cli/package.rb +2 -5
  19. data/lib/mode/commands/analyze_field.rb +20 -21
  20. data/lib/mode/commands/analyze_schema.rb +69 -48
  21. data/lib/mode/commands/connect.rb +78 -0
  22. data/lib/mode/commands/helpers.rb +54 -0
  23. data/lib/mode/commands/import.rb +209 -20
  24. data/lib/mode/commands/login.rb +111 -0
  25. data/lib/mode/config.rb +13 -33
  26. data/lib/mode/configurable.rb +46 -0
  27. data/lib/mode/connector/config.rb +31 -0
  28. data/lib/mode/connector/daemon.rb +27 -0
  29. data/lib/mode/connector/data_source.rb +75 -0
  30. data/lib/mode/connector/dataset.rb +13 -0
  31. data/lib/mode/connector/message.rb +31 -0
  32. data/lib/mode/connector/poller.rb +27 -0
  33. data/lib/mode/connector/processor.rb +58 -0
  34. data/lib/mode/connector/registrar.rb +36 -0
  35. data/lib/mode/connector/scheduler.rb +62 -0
  36. data/lib/mode/connector/selector.rb +47 -0
  37. data/lib/mode/connector/type_map.rb +45 -0
  38. data/lib/mode/connector/uploader.rb +50 -0
  39. data/lib/mode/logger.rb +202 -0
  40. data/lib/mode/version.rb +1 -1
  41. data/mode.gemspec +13 -2
  42. data/spec/api/form_spec.rb +51 -0
  43. data/spec/api/link_spec.rb +23 -0
  44. data/spec/api/request_spec.rb +111 -0
  45. data/spec/api/resource_spec.rb +70 -0
  46. data/spec/auth/access_token_spec.rb +22 -0
  47. data/spec/commands/analyze_field_spec.rb +26 -0
  48. data/spec/commands/analyze_schema_spec.rb +7 -5
  49. data/spec/commands/connect_spec.rb +80 -0
  50. data/spec/commands/helpers_spec.rb +69 -0
  51. data/spec/commands/import_spec.rb +155 -0
  52. data/spec/commands/login_spec.rb +178 -0
  53. data/spec/config_spec.rb +9 -7
  54. data/spec/connector/config_spec.rb +46 -0
  55. data/spec/connector/daemon_spec.rb +30 -0
  56. data/spec/connector/data_source_spec.rb +73 -0
  57. data/spec/connector/message_spec.rb +22 -0
  58. data/spec/connector/poller_spec.rb +26 -0
  59. data/spec/connector/processor_spec.rb +93 -0
  60. data/spec/connector/registrar_spec.rb +53 -0
  61. data/spec/connector/scheduler_spec.rb +93 -0
  62. data/spec/connector/selector_spec.rb +54 -0
  63. data/spec/connector/type_map_spec.rb +45 -0
  64. data/spec/connector/uploader_spec.rb +55 -0
  65. data/spec/fixtures/country-codes/README.md +71 -0
  66. data/spec/fixtures/country-codes/data/country-codes.csv +250 -0
  67. data/spec/fixtures/country-codes/datapackage.json +142 -0
  68. data/spec/fixtures/country-codes/scripts/get_countries_of_earth.py +370 -0
  69. data/spec/fixtures/country-codes/scripts/reorder_columns.py +8 -0
  70. data/spec/fixtures/country-codes/scripts/requirements.pip +2 -0
  71. data/spec/fixtures/espn_draft.csv +473 -1
  72. data/spec/fixtures/espn_draft/data.csv +473 -0
  73. data/spec/fixtures/espn_draft/datapackage.json +43 -0
  74. data/spec/logger_spec.rb +79 -0
  75. data/spec/spec_helper.rb +6 -1
  76. metadata +156 -19
  77. data/lib/mode/cli/setup.rb +0 -12
  78. data/lib/mode/commands/package.rb +0 -56
  79. data/lib/mode/commands/setup.rb +0 -36
  80. data/lib/mode/package_builder.rb +0 -57
  81. data/spec/commands/setup_spec.rb +0 -62
  82. data/spec/fixtures/MOCK_DATA.csv +0 -100001
  83. data/spec/fixtures/cb_clean_small.csv +0 -100000
  84. data/spec/fixtures/duplicate_keys.csv +0 -3
  85. data/spec/fixtures/format_examples.csv.txt +0 -6
  86. data/spec/fixtures/format_examples_after_excel.csv.txt +0 -1
@@ -0,0 +1,178 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Commands::Login do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+ let(:username) { 'besquared' }
6
+ let(:password) { 'password' }
7
+
8
+ let(:personal_repr) {
9
+ {
10
+ 'name' => 'personal',
11
+ 'token' => 'token123',
12
+ 'account_name' => 'besquared'
13
+ }
14
+ }
15
+
16
+ let(:secondary_repr) {
17
+ {
18
+ 'name' => 'secondary',
19
+ 'token' => 'token567',
20
+ 'account_name' => 'besquared'
21
+ }
22
+ }
23
+
24
+ let(:tokens_resource) {
25
+ Mode::API::Resource.new({
26
+ '_embedded' => {
27
+ 'access_tokens' => [
28
+ personal_repr,
29
+ secondary_repr
30
+ ]
31
+ }
32
+ })
33
+ }
34
+
35
+ let(:single_tokens_resource) {
36
+ Mode::API::Resource.new({
37
+ '_embedded' => {
38
+ 'access_tokens' => [
39
+ personal_repr
40
+ ]
41
+ }
42
+ })
43
+ }
44
+
45
+ let(:personal_token) {
46
+ Mode::Auth::AccessToken.new(
47
+ Mode::API::Resource.new(personal_repr)
48
+ )
49
+ }
50
+
51
+ let(:secondary_token) {
52
+ Mode::Auth::AccessToken.new(
53
+ Mode::API::Resource.new(secondary_repr)
54
+ )
55
+ }
56
+
57
+ before do
58
+ initialize_logger
59
+ end
60
+
61
+ it "initializes with each environment" do
62
+ [:test, :staging, :development, :production].each do |env|
63
+ Mode::Commands::Login.new(env => true)
64
+ end
65
+ end
66
+
67
+ it "executes a login process with a new config" do
68
+ command = Mode::Commands::Login.new
69
+
70
+ command.stub(:say)
71
+ command.stub(:config_dir).and_return(tmpdir)
72
+
73
+ command.should_receive(:configure_credentials).and_return([username, password])
74
+ command.should_receive(:choose_access_token).and_return(personal_token)
75
+
76
+ command.execute
77
+
78
+ config = Mode::Config.new(tmpdir)
79
+
80
+ config.username.should == 'besquared'
81
+ config.access_token.should == 'token123'
82
+ end
83
+
84
+ it "executes a login process with an existing config" do
85
+ command = Mode::Commands::Login.new
86
+
87
+ command.stub(:say)
88
+ command.stub(:config_dir).and_return(tmpdir)
89
+
90
+ Mode::Config.init(tmpdir)
91
+
92
+ command.should_receive(:configure_credentials).and_return([username, password])
93
+ command.should_receive(:choose_access_token).and_return(personal_token)
94
+
95
+ command.execute
96
+
97
+ config = Mode::Config.new(tmpdir)
98
+
99
+ config.username.should == 'besquared'
100
+ config.access_token.should == 'token123'
101
+ end
102
+
103
+ it "rescues and displays errors during the login process" do
104
+ command = Mode::Commands::Login.new
105
+
106
+ command.stub(:config_dir).and_return(tmpdir)
107
+
108
+ error = StandardError.new("Expected Break")
109
+ command.should_receive(:configure_credentials).and_raise(error)
110
+
111
+ command.should_receive(:say).with("Expected Break")
112
+
113
+ command.execute
114
+ end
115
+
116
+ it "configures API with credentials" do
117
+ command = Mode::Commands::Login.new(:test => true)
118
+
119
+ command.send(:configure_api, 'besquared', 'password')
120
+
121
+ Mode::API::Request.environment.should == :test
122
+ end
123
+
124
+ it "requests credentials and configures API" do
125
+ command = Mode::Commands::Login.new(:test => true)
126
+
127
+ command.should_receive(:configure_api)
128
+ command.should_receive(:ask).and_return('besquared', 'password')
129
+
130
+ username, password = command.send(:configure_credentials)
131
+
132
+ username.should == 'besquared'
133
+ password.should == 'password'
134
+ end
135
+
136
+ it "raises when credentials are invalid" do
137
+ command = Mode::Commands::Login.new(:test => true)
138
+
139
+ response = double(:response, :status => 401)
140
+ Mode::API::Request.should_receive(:get).with(:access_tokens).and_return(response)
141
+ expect { command.send(:fetch_access_tokens) }.to raise_error
142
+ end
143
+
144
+ it "fetches access tokens from the API" do
145
+ command = Mode::Commands::Login.new(:test => true)
146
+ Mode::API::Request.should_receive(:get).with(:access_tokens).and_return(tokens_resource)
147
+
148
+ tokens = command.send(:fetch_access_tokens)
149
+
150
+ tokens.length.should == 2
151
+ tokens.first.token.should == 'token123'
152
+ tokens.last.token.should == 'token567'
153
+ end
154
+
155
+ it "chooses the first token if there is only one" do
156
+ command = Mode::Commands::Login.new(:test => true)
157
+ Mode::API::Request.should_receive(:get).with(:access_tokens).and_return(single_tokens_resource)
158
+
159
+ command.should_not_receive(:ask)
160
+
161
+ chosen_token = command.send(:choose_access_token)
162
+
163
+ chosen_token.should_not == nil
164
+ chosen_token.token.should == 'token123'
165
+ end
166
+
167
+ it "prompts the user to choose a token" do
168
+ command = Mode::Commands::Login.new(:test => true)
169
+ Mode::API::Request.should_receive(:get).with(:access_tokens).and_return(tokens_resource)
170
+
171
+ command.should_receive(:ask).and_return("2")
172
+
173
+ chosen_token = command.send(:choose_access_token)
174
+
175
+ chosen_token.should_not == nil
176
+ chosen_token.token.should == 'token567'
177
+ end
178
+ end
@@ -1,12 +1,14 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe Mode::Config do
4
- before(:each) do
5
- @tmpdir = Dir.mktmpdir
6
- end
4
+ let(:tmpdir) { Dir.mktmpdir }
7
5
 
8
6
  it "should initialize" do
9
- config = Mode::Config.init(@tmpdir)
7
+ Mode::Config.exists?(tmpdir).should == false
8
+
9
+ config = Mode::Config.init(tmpdir)
10
+ Mode::Config.exists?(tmpdir).should == true
11
+
10
12
  config.username.should == nil
11
13
  config.access_token.should == nil
12
14
  end
@@ -16,7 +18,7 @@ describe Mode::Config do
16
18
  end
17
19
 
18
20
  it "should save" do
19
- config = Mode::Config.init(@tmpdir)
21
+ config = Mode::Config.init(tmpdir)
20
22
  config.username = 'besquared'
21
23
  config.access_token = 'token'
22
24
  config.save
@@ -25,10 +27,10 @@ describe Mode::Config do
25
27
  end
26
28
 
27
29
  it "should have FILENAME" do
28
- Mode::Config::FILENAME.should == '.mode.yml'
30
+ Mode::Config::default_filename.should == 'config.yml'
29
31
  end
30
32
 
31
33
  it "should generate full path" do
32
- Mode::Config.full_path(@tmpdir).should == File.join(@tmpdir, '.mode.yml')
34
+ Mode::Config.full_path(tmpdir).should == File.join(tmpdir, 'config.yml')
33
35
  end
34
36
  end
@@ -0,0 +1,46 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Config do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+
6
+ before do
7
+ initialize_logger
8
+ end
9
+
10
+ it "should initialize" do
11
+ Mode::Connector::Config.exists?(tmpdir).should == false
12
+ config = Mode::Connector::Config.init(tmpdir)
13
+ Mode::Connector::Config.exists?(tmpdir).should == true
14
+
15
+ config.data_sources.should == []
16
+ end
17
+
18
+ it "should initialize with data sources" do
19
+ config = Mode::Connector::Config.init(tmpdir)
20
+ config.data_sources << Mode::Connector::DataSource.new('test', {})
21
+ config.save
22
+
23
+ config = Mode::Connector::Config.new(tmpdir)
24
+ config.data_sources.length.should == 1
25
+ end
26
+
27
+ it "should raise if config file doesn't exist" do
28
+ expect { config = Mode::Connector::Config.new('fake.yml') }.to raise_error
29
+ end
30
+
31
+ it "should save" do
32
+ config = Mode::Connector::Config.init(tmpdir)
33
+ config.data_sources << Mode::Connector::DataSource.new('test', {})
34
+ config.save
35
+
36
+ File.read(config.path).should == config.send(:to_yaml)
37
+ end
38
+
39
+ it "should have FILENAME" do
40
+ Mode::Connector::Config::default_filename.should == 'connect.yml'
41
+ end
42
+
43
+ it "should generate full path" do
44
+ Mode::Connector::Config.full_path(tmpdir).should == File.join(tmpdir, 'connect.yml')
45
+ end
46
+ end
@@ -0,0 +1,30 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Daemon do
4
+ let(:working_dir) { Dir.mktmpdir }
5
+ let(:scheduler) { double(:scheduler, :start! => true, :stop! => true) }
6
+
7
+ before do
8
+ initialize_logger
9
+ end
10
+
11
+ it "starts a daemon" do
12
+ daemon = Mode::Connector::Daemon.new(:working_dir => working_dir)
13
+ Mode::Connector::Scheduler.should_receive(:new).and_return(scheduler)
14
+
15
+ begin
16
+ daemon.start([5, []])
17
+ ensure
18
+ daemon.stop
19
+ end
20
+ end
21
+
22
+ it "logs errors on failure" do
23
+ daemon = Mode::Connector::Daemon.new(:working_dir => working_dir)
24
+ Mode::Connector::Scheduler.should_receive(:new).and_raise(StandardError.new("BREAK"))
25
+
26
+ Mode::Logger.instance.should_receive(:error)
27
+
28
+ daemon.start([5, []])
29
+ end
30
+ end
@@ -0,0 +1,73 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::DataSource do
4
+ let(:props) {
5
+ {
6
+ 'adapter' => 'postgres',
7
+ 'host' => 'localhost',
8
+ 'username' => 'postgres',
9
+ 'database' => 'warehouse',
10
+ 'password' => nil
11
+ }
12
+ }
13
+
14
+ let(:redprops) {
15
+ {
16
+ 'adapter' => 'redshift',
17
+ 'host' => 'localhost',
18
+ 'username' => 'postgres',
19
+ 'database' => 'warehouse',
20
+ 'password' => nil
21
+ }
22
+ }
23
+
24
+ it "initializes with props" do
25
+ source = Mode::Connector::DataSource.new('dev', props)
26
+ source.adapter.should == 'postgres'
27
+ source.host.should == 'localhost'
28
+ source.username.should == 'postgres'
29
+ source.database.should == 'warehouse'
30
+ source.password.should == nil
31
+ end
32
+
33
+ before do
34
+ initialize_logger
35
+ end
36
+
37
+ it "has a connection url" do
38
+ source = Mode::Connector::DataSource.new('dev', props)
39
+ source.send(:connection_url).should == "postgres://postgres@localhost/warehouse"
40
+ end
41
+
42
+ it "has a connection" do
43
+ source = Mode::Connector::DataSource.new('dev', props)
44
+ Sequel.should_receive(:connect).and_return(:connect)
45
+ source.connection.should == :connect
46
+ end
47
+
48
+ it 'has a connection for redshift' do
49
+ source = Mode::Connector::DataSource.new('dev', redprops)
50
+ Sequel.should_receive(:connect).and_return(:connect)
51
+ source.connection.should == :connect
52
+ end
53
+
54
+ it 'has a connection dataset' do
55
+ source = Mode::Connector::DataSource.new('dev', props)
56
+
57
+ dataset = source.send(:connection_dataset, "SELECT * FROM fake")
58
+
59
+ dataset.should_not == nil
60
+ end
61
+
62
+ it "fetches results from dataset" do
63
+ source = Mode::Connector::DataSource.new('dev', props)
64
+
65
+ dataset = double(:dataset)
66
+ dataset.should_receive(:each).and_yield({'cnt' => 5})
67
+ source.should_receive(:connection_dataset).with("SELECT 1").and_return(dataset)
68
+
69
+ source.select("SELECT 1") do |row|
70
+ row['cnt'].should == 5
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Message do
4
+ let(:resource) {
5
+ Mode::API::Resource.new({
6
+ 'type' => 'query',
7
+ 'name' => 'fake',
8
+ 'query' => 'query'
9
+ })
10
+ }
11
+
12
+ before do
13
+ initialize_logger
14
+ end
15
+
16
+ it "initializes with resource" do
17
+ message = Mode::Connector::Message.new(resource)
18
+ message.type.should == 'query'
19
+ message.name.should == 'fake'
20
+ message.query.should == 'query'
21
+ end
22
+ end
@@ -0,0 +1,26 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Poller do
4
+ let(:path) {
5
+ Mode::API::Request.data_source_connection_messages_path
6
+ }
7
+
8
+ before do
9
+ initialize_logger
10
+ end
11
+
12
+ it "initializes and performs poll" do
13
+ poller = Mode::Connector::Poller.new
14
+
15
+ resource = Mode::API::Resource.new({
16
+ '_embedded' => {
17
+ 'messages' => [{}, {}]
18
+ }
19
+ })
20
+ Mode::API::Request.should_receive(:get).with(path).and_return(resource)
21
+
22
+ poller.perform! do |message|
23
+ message.should be_instance_of(Mode::Connector::Message)
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,93 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Connector::Processor do
4
+ let(:data_sources) {
5
+ [
6
+ Mode::Connector::DataSource.new('testdb', {
7
+ 'adapter' => 'postgres',
8
+ 'host' => 'localhost',
9
+ 'username' => 'postgres',
10
+ 'database' => 'warehouse',
11
+ 'password' => nil
12
+ })
13
+ ]
14
+ }
15
+
16
+ let(:execution_path) { '/execution' }
17
+
18
+ let(:resource) {
19
+ Mode::API::Resource.new({
20
+ 'type' => 'query',
21
+ 'query' => 'SELECT 1',
22
+ '_embedded' => {
23
+ 'data_source' => {
24
+ 'name' => 'testdb'
25
+ }
26
+ },
27
+ '_links' => {
28
+ 'execution' => {
29
+ 'href' => execution_path
30
+ }
31
+ }
32
+ })
33
+ }
34
+
35
+ let(:message) {
36
+ Mode::Connector::Message.new(resource)
37
+ }
38
+
39
+ before do
40
+ initialize_logger
41
+ end
42
+
43
+ it "initializes with message and data sources" do
44
+ processor = Mode::Connector::Processor.new(message, data_sources)
45
+
46
+ processor.message.should_not == nil
47
+ processor.data_sources.should_not == nil
48
+ end
49
+
50
+ it "selects a dataset" do
51
+ processor = Mode::Connector::Processor.new(message, data_sources)
52
+
53
+ selector = double(:selector, :perform! => true)
54
+ Mode::Connector::Selector.should_receive(:new).with(
55
+ message.query, data_sources.first).and_return(selector)
56
+
57
+ processor.send(:select!)
58
+ end
59
+
60
+ it "sends a dataset" do
61
+ processor = Mode::Connector::Processor.new(message, data_sources)
62
+
63
+ dataset = double(:dataset)
64
+ uploader = double(:uploader, :perform! => true)
65
+ Mode::Connector::Uploader.should_receive(:new).with(execution_path, dataset).and_return(uploader)
66
+
67
+ processor.send(:send!, dataset)
68
+ end
69
+
70
+ it "sends an error" do
71
+ processor = Mode::Connector::Processor.new(message, data_sources)
72
+
73
+ Mode::Connector::Uploader.should_receive(:error!).with(
74
+ execution_path, "message", "details").and_return(true)
75
+
76
+ processor.send(:error!, "message", "details")
77
+ end
78
+
79
+ it "performs a send" do
80
+ processor = Mode::Connector::Processor.new(message, data_sources)
81
+ processor.should_receive(:select!).and_return(true)
82
+ processor.should_receive(:send!).and_return(true)
83
+ processor.perform!
84
+ end
85
+
86
+ it "performs an error" do
87
+ processor = Mode::Connector::Processor.new(message, data_sources)
88
+ processor.should_receive(:select!).and_raise(StandardError.new("BREAK!"))
89
+ processor.should_receive(:error!).and_return(true)
90
+
91
+ processor.perform!
92
+ end
93
+ end