mode 0.0.5 → 0.0.7

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -1
  3. data/README.md +17 -22
  4. data/bin/mode +1 -1
  5. data/lib/mode.rb +34 -6
  6. data/lib/mode/api/form.rb +53 -0
  7. data/lib/mode/api/link.rb +31 -0
  8. data/lib/mode/api/request.rb +181 -0
  9. data/lib/mode/api/resource.rb +67 -0
  10. data/lib/mode/auth/access_token.rb +23 -0
  11. data/lib/mode/cli.rb +3 -3
  12. data/lib/mode/cli/analyze.rb +1 -1
  13. data/lib/mode/cli/base.rb +5 -0
  14. data/lib/mode/cli/connect.rb +18 -0
  15. data/lib/mode/cli/helpers.rb +0 -9
  16. data/lib/mode/cli/import.rb +9 -38
  17. data/lib/mode/cli/login.rb +13 -0
  18. data/lib/mode/cli/package.rb +2 -5
  19. data/lib/mode/commands/analyze_field.rb +20 -21
  20. data/lib/mode/commands/analyze_schema.rb +69 -48
  21. data/lib/mode/commands/connect.rb +78 -0
  22. data/lib/mode/commands/helpers.rb +54 -0
  23. data/lib/mode/commands/import.rb +209 -20
  24. data/lib/mode/commands/login.rb +111 -0
  25. data/lib/mode/config.rb +13 -33
  26. data/lib/mode/configurable.rb +46 -0
  27. data/lib/mode/connector/config.rb +31 -0
  28. data/lib/mode/connector/daemon.rb +27 -0
  29. data/lib/mode/connector/data_source.rb +75 -0
  30. data/lib/mode/connector/dataset.rb +13 -0
  31. data/lib/mode/connector/message.rb +31 -0
  32. data/lib/mode/connector/poller.rb +27 -0
  33. data/lib/mode/connector/processor.rb +58 -0
  34. data/lib/mode/connector/registrar.rb +36 -0
  35. data/lib/mode/connector/scheduler.rb +62 -0
  36. data/lib/mode/connector/selector.rb +47 -0
  37. data/lib/mode/connector/type_map.rb +45 -0
  38. data/lib/mode/connector/uploader.rb +50 -0
  39. data/lib/mode/logger.rb +202 -0
  40. data/lib/mode/version.rb +1 -1
  41. data/mode.gemspec +13 -2
  42. data/spec/api/form_spec.rb +51 -0
  43. data/spec/api/link_spec.rb +23 -0
  44. data/spec/api/request_spec.rb +111 -0
  45. data/spec/api/resource_spec.rb +70 -0
  46. data/spec/auth/access_token_spec.rb +22 -0
  47. data/spec/commands/analyze_field_spec.rb +26 -0
  48. data/spec/commands/analyze_schema_spec.rb +7 -5
  49. data/spec/commands/connect_spec.rb +80 -0
  50. data/spec/commands/helpers_spec.rb +69 -0
  51. data/spec/commands/import_spec.rb +155 -0
  52. data/spec/commands/login_spec.rb +178 -0
  53. data/spec/config_spec.rb +9 -7
  54. data/spec/connector/config_spec.rb +46 -0
  55. data/spec/connector/daemon_spec.rb +30 -0
  56. data/spec/connector/data_source_spec.rb +73 -0
  57. data/spec/connector/message_spec.rb +22 -0
  58. data/spec/connector/poller_spec.rb +26 -0
  59. data/spec/connector/processor_spec.rb +93 -0
  60. data/spec/connector/registrar_spec.rb +53 -0
  61. data/spec/connector/scheduler_spec.rb +93 -0
  62. data/spec/connector/selector_spec.rb +54 -0
  63. data/spec/connector/type_map_spec.rb +45 -0
  64. data/spec/connector/uploader_spec.rb +55 -0
  65. data/spec/fixtures/country-codes/README.md +71 -0
  66. data/spec/fixtures/country-codes/data/country-codes.csv +250 -0
  67. data/spec/fixtures/country-codes/datapackage.json +142 -0
  68. data/spec/fixtures/country-codes/scripts/get_countries_of_earth.py +370 -0
  69. data/spec/fixtures/country-codes/scripts/reorder_columns.py +8 -0
  70. data/spec/fixtures/country-codes/scripts/requirements.pip +2 -0
  71. data/spec/fixtures/espn_draft.csv +473 -1
  72. data/spec/fixtures/espn_draft/data.csv +473 -0
  73. data/spec/fixtures/espn_draft/datapackage.json +43 -0
  74. data/spec/logger_spec.rb +79 -0
  75. data/spec/spec_helper.rb +6 -1
  76. metadata +156 -19
  77. data/lib/mode/cli/setup.rb +0 -12
  78. data/lib/mode/commands/package.rb +0 -56
  79. data/lib/mode/commands/setup.rb +0 -36
  80. data/lib/mode/package_builder.rb +0 -57
  81. data/spec/commands/setup_spec.rb +0 -62
  82. data/spec/fixtures/MOCK_DATA.csv +0 -100001
  83. data/spec/fixtures/cb_clean_small.csv +0 -100000
  84. data/spec/fixtures/duplicate_keys.csv +0 -3
  85. data/spec/fixtures/format_examples.csv.txt +0 -6
  86. data/spec/fixtures/format_examples_after_excel.csv.txt +0 -1
@@ -0,0 +1,70 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::API::Resource do
4
+ let(:links) {
5
+ {
6
+ 'self' => {
7
+ 'templated' => true,
8
+ 'href' => '/path/to/thing'
9
+ }
10
+ }
11
+ }
12
+
13
+ let(:forms) {
14
+ {
15
+ 'create' => {
16
+ 'method' => 'post',
17
+ 'action' => '/widgets',
18
+ 'input' => {
19
+ 'widgets' => {
20
+ 'type' => 'text',
21
+ 'value' => '500'
22
+ }
23
+ }
24
+ }
25
+ }
26
+ }
27
+
28
+ let(:embedded) {
29
+ {
30
+ 'messages' => [{}],
31
+ 'account' => {}
32
+ }
33
+ }
34
+
35
+ let(:content) {
36
+ {
37
+ 'thing' => 'value',
38
+ '_links' => links,
39
+ '_forms' => forms,
40
+ '_embedded' => embedded
41
+ }
42
+ }
43
+
44
+ before do
45
+ initialize_logger
46
+ end
47
+
48
+ it "initializes with content" do
49
+ resource = Mode::API::Resource.new(content)
50
+
51
+ resource.thing.should == 'value'
52
+
53
+ expect { resource.thing2 }.to raise_error
54
+
55
+ link = resource.links('self')
56
+ link.href.should == links['self']['href']
57
+
58
+ form = resource.forms('create')
59
+ form.action.should == forms['create']['action']
60
+
61
+ messages = resource.embedded('messages')
62
+ messages.length.should == 1
63
+ messages.first.should be_instance_of(Mode::API::Resource)
64
+ messages.first.content.should == embedded['messages'].first
65
+
66
+ account = resource.embedded('account')
67
+ account.should be_instance_of(Mode::API::Resource)
68
+ account.content.should == embedded['account']
69
+ end
70
+ end
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Auth::AccessToken do
4
+ let(:resource) {
5
+ Mode::API::Resource.new({
6
+ 'name' => 'Personal',
7
+ 'token' => 'token123',
8
+ 'account_name' => 'besquared'
9
+ })
10
+ }
11
+
12
+ before do
13
+ initialize_logger
14
+ end
15
+
16
+ it "initializes with resource" do
17
+ token = Mode::Auth::AccessToken.new(resource)
18
+ token.name.should == 'Personal'
19
+ token.token.should == 'token123'
20
+ token.account_name.should == 'besquared'
21
+ end
22
+ end
@@ -0,0 +1,26 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Commands::AnalyzeField do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+
6
+ before do
7
+ initialize_logger
8
+ end
9
+
10
+ it "should analyze a csv" do
11
+ path = fixture_path('espn_draft.csv')
12
+ command = Mode::Commands::AnalyzeField.new(path, 1)
13
+
14
+ command.stub(:puts).and_return(true)
15
+
16
+ command.execute
17
+ end
18
+
19
+ it "should error out if the csv file doesn't exist" do
20
+ command = Mode::Commands::AnalyzeField.new("fake.yml", 1)
21
+
22
+ command.should_receive(:puts)
23
+
24
+ command.execute
25
+ end
26
+ end
@@ -1,15 +1,17 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe Mode::Commands::AnalyzeSchema do
4
- before(:each) do
5
- @tmpdir = Dir.mktmpdir
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+
6
+ before do
7
+ initialize_logger
6
8
  end
7
9
 
8
10
  it "should analyze a csv" do
9
- path = data_path('espn_draft.csv')
11
+ path = fixture_path('espn_draft.csv')
10
12
  command = Mode::Commands::AnalyzeSchema.new(path)
11
13
 
12
- command.stub(:say).and_return(true)
14
+ command.stub(:puts).and_return(true)
13
15
 
14
16
  command.execute
15
17
  end
@@ -17,7 +19,7 @@ describe Mode::Commands::AnalyzeSchema do
17
19
  it "should error out if the csv file doesn't exist" do
18
20
  command = Mode::Commands::AnalyzeSchema.new("fake.yml", keys: [0,1])
19
21
 
20
- command.should_not_receive(:say)
22
+ command.should_receive(:puts)
21
23
 
22
24
  command.execute
23
25
  end
@@ -0,0 +1,80 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Commands::Connect do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+ let(:config) { Mode::Connector::Config.init(tmpdir) }
6
+
7
+ before do
8
+ initialize_logger
9
+ end
10
+
11
+ it "initializes with command and connect config" do
12
+ connect = Mode::Commands::Connect.new('start')
13
+
14
+ connect.command.should == 'start'
15
+ connect.concurrency.should == 4
16
+ connect.configuration.should_not == nil
17
+ end
18
+
19
+ it "registers connector" do
20
+ connect = Mode::Commands::Connect.new('start')
21
+
22
+ registrar = double(:registrar, :perform! => true)
23
+ Mode::Connector::Registrar.should_receive(:new).and_return(registrar)
24
+
25
+ connect.send(:register!)
26
+ end
27
+
28
+ it "starts the daemon" do
29
+ connect = Mode::Commands::Connect.new('start')
30
+
31
+ Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
32
+
33
+ connect.send(:spawn_command!, 'start')
34
+ end
35
+
36
+ it 'restarts the daemon' do
37
+ connect = Mode::Commands::Connect.new('start')
38
+
39
+ Mode::Connector::Daemon.should_receive(:spawn!).twice.and_return(true)
40
+
41
+ connect.send(:spawn_restart!)
42
+ end
43
+
44
+ it 'spawns with the start command' do
45
+ connect = Mode::Commands::Connect.new('start')
46
+ Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
47
+
48
+ connect.send(:spawn!)
49
+ end
50
+
51
+ it 'spawns with the restart command' do
52
+ connect = Mode::Commands::Connect.new('restart')
53
+ Mode::Connector::Daemon.should_receive(:spawn!).twice.and_return(true)
54
+
55
+ connect.send(:spawn!)
56
+ end
57
+
58
+ it 'spawns with the stop command' do
59
+ connect = Mode::Commands::Connect.new('stop')
60
+ Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
61
+
62
+ connect.send(:spawn!)
63
+ end
64
+
65
+ it 'raises on spawn if the command is unrecognized' do
66
+ connect = Mode::Commands::Connect.new('fake')
67
+ Mode::Connector::Daemon.should_not_receive(:spawn!)
68
+
69
+ expect { connect.send(:spawn!) }.to raise_error
70
+ end
71
+
72
+ it 'executes the connector' do
73
+ connect = Mode::Commands::Connect.new('start')
74
+
75
+ connect.should_receive(:register!)
76
+ connect.should_receive(:spawn!)
77
+
78
+ connect.execute
79
+ end
80
+ end
@@ -0,0 +1,69 @@
1
+ require 'spec_helper'
2
+
3
+ class CommandClass
4
+ include Mode::Commands::Helpers
5
+ end
6
+
7
+ describe Mode::Commands::Helpers do
8
+ let(:tmpdir) { Dir.mktmpdir }
9
+ let(:command) { CommandClass.new }
10
+
11
+ before do
12
+ initialize_logger
13
+ end
14
+
15
+ it "provides a config dir" do
16
+ command.config_dir.should == File.expand_path("~/.mode/")
17
+ end
18
+
19
+ it "provides a config path" do
20
+ command.config_path.should == Mode::Config.full_path("~/.mode/")
21
+ end
22
+
23
+ it "provides config validation" do
24
+ command.stub(:config_dir).and_return(tmpdir)
25
+
26
+ expect { command.require_config! }.to raise_error
27
+
28
+ config = Mode::Config.init(tmpdir)
29
+
30
+ command.require_config!
31
+ end
32
+
33
+ it "provides credential validation" do
34
+ command.stub(:config_dir).and_return(tmpdir)
35
+
36
+ # raise for no config
37
+ expect { command.require_credentials! }.to raise_error
38
+
39
+ config = Mode::Config.init(tmpdir)
40
+
41
+ # raise for credentials not found
42
+ expect { command.require_credentials! }.to raise_error
43
+
44
+ config.username = 'besquared'
45
+ config.access_token = 'access_token'
46
+ config.save
47
+
48
+ command.require_credentials!
49
+ end
50
+
51
+ it "provides api request configuration" do
52
+ command.configure_api_requests!
53
+ Mode::API::Request.valid?.should == true
54
+ end
55
+
56
+ it "provides connect config validation" do
57
+ command.stub(:config_dir).and_return(tmpdir)
58
+
59
+ # raise for no config
60
+ expect { command.require_connect_config! }.to raise_error
61
+
62
+ config = Mode::Connector::Config.init(tmpdir)
63
+
64
+ config.data_sources << Mode::Connector::DataSource.new('test', {})
65
+ config.save
66
+
67
+ command.require_connect_config!
68
+ end
69
+ end
@@ -0,0 +1,155 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Commands::Import do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+ let(:packages_path) { Mode::API::Request.packages_path }
6
+
7
+ before do
8
+ initialize_logger
9
+ end
10
+
11
+ it "generates target resource path with a csv extension" do
12
+ source = fixture_path("espn_draft/data.csv")
13
+ path = Mode::Commands::Import.send(:target_resource_path, source)
14
+
15
+ path.should == "data/index.csv"
16
+ end
17
+
18
+ it "generates target resource path with a non-csv extension" do
19
+ source = fixture_path("espn_draft.json")
20
+ path = Mode::Commands::Import.send(:target_resource_path, source)
21
+
22
+ path.should == "data/index.json"
23
+ end
24
+
25
+ describe 'finding a package from a source file' do
26
+ it 'returns the package if the file is adjacent to datapackage.json' do
27
+ source = fixture_path("espn_draft/data.csv")
28
+ package = Mode::Commands::Import.find_package(source)
29
+
30
+ package.base_path.should == fixture_path("espn_draft")
31
+ end
32
+
33
+ it 'returns the package if the file is inside of the data subdirectory' do
34
+ source = fixture_path("country-codes/data/country-codes.csv")
35
+ package = Mode::Commands::Import.find_package(source)
36
+
37
+ package.base_path.should == fixture_path("country-codes")
38
+ end
39
+
40
+ it 'returns nil if a package is not found' do
41
+ source = fixture_path("espn_draft.csv")
42
+ Mode::Commands::Import.find_package(source).should == nil
43
+ end
44
+ end
45
+
46
+ describe 'finding a package from a source directory' do
47
+ it 'returns the package if it exists in the directory' do
48
+ source = fixture_path("espn_draft")
49
+ package = Mode::Commands::Import.find_package(source)
50
+
51
+ package.base_path.should == fixture_path("espn_draft")
52
+ end
53
+
54
+ it 'returns nil if the package is not found in the directory' do
55
+ source = fixture_path
56
+
57
+ Mode::Commands::Import.find_package(source).should == nil
58
+ end
59
+ end
60
+
61
+ describe 'building a package from a source file' do
62
+ it 'build the package' do
63
+ source = fixture_path("espn_draft.csv")
64
+ package = Mode::Commands::Import.build_package(source, :dest => tmpdir)
65
+
66
+ package.base_path.should == tmpdir
67
+ end
68
+
69
+ it 'returns nil if the file does not exist' do
70
+ source = fixture_path('nope')
71
+ Mode::Commands::Import.build_package(source).should == nil
72
+ end
73
+ end
74
+
75
+ describe 'finding or building a package from a source file' do
76
+ it 'returns the package and resource name for a file in a datapackage' do
77
+ source = fixture_path("espn_draft/data.csv")
78
+ package = Mode::Commands::Import.find_or_build_package(source)
79
+
80
+ package.resources.first.path.should == 'data.csv'
81
+ package.base_path.should == fixture_path("espn_draft")
82
+ end
83
+
84
+ it 'returns the package and resource name for a file outisde a datapackage' do
85
+ source = fixture_path("espn_draft.csv")
86
+ package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
87
+
88
+ package.base_path.should == tmpdir
89
+ package.resources.first.path.should == 'data/index.csv'
90
+ end
91
+ end
92
+
93
+ describe 'finding or building a package from a source directory' do
94
+ it 'returns the package and resource name' do
95
+ source = fixture_path("espn_draft")
96
+ package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
97
+
98
+ package.resources.first.path.should == 'data.csv'
99
+ package.base_path.should == fixture_path("espn_draft")
100
+ end
101
+ end
102
+
103
+ describe 'uploading a package' do
104
+ it 'uploads a package' do
105
+ Mode::API::Request.should_receive(:post).and_return(true)
106
+
107
+ source = fixture_path("espn_draft.csv")
108
+ package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
109
+
110
+ import = Mode::Commands::Import.new(source, 'besquared', 'espn_draft')
111
+
112
+ import.upload_package(package).should == true
113
+ end
114
+ end
115
+
116
+ describe 'executing an import' do
117
+ it 'finds and upload a package' do
118
+ Mode::API::Request.should_receive(:post).with(packages_path, :package => {
119
+ :contents => {
120
+ 'datapackage.json' => an_instance_of(String),
121
+ 'data/index.csv' => an_instance_of(Faraday::UploadIO)
122
+ }, :name => 'espn_draft'
123
+ }).and_return(:package)
124
+
125
+ source = fixture_path("espn_draft/data.csv")
126
+ import = Mode::Commands::Import.new(source, 'besquared', 'espn_draft')
127
+
128
+ resource = double(:resource)
129
+ import_params = { 'table_name' => 'espn_draft' }
130
+ import.should_receive(:parse_uploaded_response).with(:package).and_return(resource)
131
+ import.should_receive(:submit_import_form).with(resource, import_params).and_return(true)
132
+
133
+ import.execute
134
+ end
135
+
136
+ it 'finds and uploads country codes package' do
137
+ Mode::API::Request.should_receive(:post).with(packages_path, :package => {
138
+ :contents => {
139
+ 'datapackage.json' => an_instance_of(String),
140
+ 'data/index.csv' => an_instance_of(Faraday::UploadIO)
141
+ }, :name => 'country_codes'
142
+ }).and_return(:package)
143
+
144
+ source = fixture_path("country-codes/data/country-codes.csv")
145
+ import = Mode::Commands::Import.new(source, 'besquared', 'country_codes')
146
+
147
+ resource = double(:resource)
148
+ import_params = { 'table_name' => 'country_codes' }
149
+ import.should_receive(:parse_uploaded_response).with(:package).and_return(resource)
150
+ import.should_receive(:submit_import_form).with(resource, import_params).and_return(true)
151
+
152
+ import.execute
153
+ end
154
+ end
155
+ end