mode 0.0.5 → 0.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -1
  3. data/README.md +17 -22
  4. data/bin/mode +1 -1
  5. data/lib/mode.rb +34 -6
  6. data/lib/mode/api/form.rb +53 -0
  7. data/lib/mode/api/link.rb +31 -0
  8. data/lib/mode/api/request.rb +181 -0
  9. data/lib/mode/api/resource.rb +67 -0
  10. data/lib/mode/auth/access_token.rb +23 -0
  11. data/lib/mode/cli.rb +3 -3
  12. data/lib/mode/cli/analyze.rb +1 -1
  13. data/lib/mode/cli/base.rb +5 -0
  14. data/lib/mode/cli/connect.rb +18 -0
  15. data/lib/mode/cli/helpers.rb +0 -9
  16. data/lib/mode/cli/import.rb +9 -38
  17. data/lib/mode/cli/login.rb +13 -0
  18. data/lib/mode/cli/package.rb +2 -5
  19. data/lib/mode/commands/analyze_field.rb +20 -21
  20. data/lib/mode/commands/analyze_schema.rb +69 -48
  21. data/lib/mode/commands/connect.rb +78 -0
  22. data/lib/mode/commands/helpers.rb +54 -0
  23. data/lib/mode/commands/import.rb +209 -20
  24. data/lib/mode/commands/login.rb +111 -0
  25. data/lib/mode/config.rb +13 -33
  26. data/lib/mode/configurable.rb +46 -0
  27. data/lib/mode/connector/config.rb +31 -0
  28. data/lib/mode/connector/daemon.rb +27 -0
  29. data/lib/mode/connector/data_source.rb +75 -0
  30. data/lib/mode/connector/dataset.rb +13 -0
  31. data/lib/mode/connector/message.rb +31 -0
  32. data/lib/mode/connector/poller.rb +27 -0
  33. data/lib/mode/connector/processor.rb +58 -0
  34. data/lib/mode/connector/registrar.rb +36 -0
  35. data/lib/mode/connector/scheduler.rb +62 -0
  36. data/lib/mode/connector/selector.rb +47 -0
  37. data/lib/mode/connector/type_map.rb +45 -0
  38. data/lib/mode/connector/uploader.rb +50 -0
  39. data/lib/mode/logger.rb +202 -0
  40. data/lib/mode/version.rb +1 -1
  41. data/mode.gemspec +13 -2
  42. data/spec/api/form_spec.rb +51 -0
  43. data/spec/api/link_spec.rb +23 -0
  44. data/spec/api/request_spec.rb +111 -0
  45. data/spec/api/resource_spec.rb +70 -0
  46. data/spec/auth/access_token_spec.rb +22 -0
  47. data/spec/commands/analyze_field_spec.rb +26 -0
  48. data/spec/commands/analyze_schema_spec.rb +7 -5
  49. data/spec/commands/connect_spec.rb +80 -0
  50. data/spec/commands/helpers_spec.rb +69 -0
  51. data/spec/commands/import_spec.rb +155 -0
  52. data/spec/commands/login_spec.rb +178 -0
  53. data/spec/config_spec.rb +9 -7
  54. data/spec/connector/config_spec.rb +46 -0
  55. data/spec/connector/daemon_spec.rb +30 -0
  56. data/spec/connector/data_source_spec.rb +73 -0
  57. data/spec/connector/message_spec.rb +22 -0
  58. data/spec/connector/poller_spec.rb +26 -0
  59. data/spec/connector/processor_spec.rb +93 -0
  60. data/spec/connector/registrar_spec.rb +53 -0
  61. data/spec/connector/scheduler_spec.rb +93 -0
  62. data/spec/connector/selector_spec.rb +54 -0
  63. data/spec/connector/type_map_spec.rb +45 -0
  64. data/spec/connector/uploader_spec.rb +55 -0
  65. data/spec/fixtures/country-codes/README.md +71 -0
  66. data/spec/fixtures/country-codes/data/country-codes.csv +250 -0
  67. data/spec/fixtures/country-codes/datapackage.json +142 -0
  68. data/spec/fixtures/country-codes/scripts/get_countries_of_earth.py +370 -0
  69. data/spec/fixtures/country-codes/scripts/reorder_columns.py +8 -0
  70. data/spec/fixtures/country-codes/scripts/requirements.pip +2 -0
  71. data/spec/fixtures/espn_draft.csv +473 -1
  72. data/spec/fixtures/espn_draft/data.csv +473 -0
  73. data/spec/fixtures/espn_draft/datapackage.json +43 -0
  74. data/spec/logger_spec.rb +79 -0
  75. data/spec/spec_helper.rb +6 -1
  76. metadata +156 -19
  77. data/lib/mode/cli/setup.rb +0 -12
  78. data/lib/mode/commands/package.rb +0 -56
  79. data/lib/mode/commands/setup.rb +0 -36
  80. data/lib/mode/package_builder.rb +0 -57
  81. data/spec/commands/setup_spec.rb +0 -62
  82. data/spec/fixtures/MOCK_DATA.csv +0 -100001
  83. data/spec/fixtures/cb_clean_small.csv +0 -100000
  84. data/spec/fixtures/duplicate_keys.csv +0 -3
  85. data/spec/fixtures/format_examples.csv.txt +0 -6
  86. data/spec/fixtures/format_examples_after_excel.csv.txt +0 -1
@@ -0,0 +1,70 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::API::Resource do
4
+ let(:links) {
5
+ {
6
+ 'self' => {
7
+ 'templated' => true,
8
+ 'href' => '/path/to/thing'
9
+ }
10
+ }
11
+ }
12
+
13
+ let(:forms) {
14
+ {
15
+ 'create' => {
16
+ 'method' => 'post',
17
+ 'action' => '/widgets',
18
+ 'input' => {
19
+ 'widgets' => {
20
+ 'type' => 'text',
21
+ 'value' => '500'
22
+ }
23
+ }
24
+ }
25
+ }
26
+ }
27
+
28
+ let(:embedded) {
29
+ {
30
+ 'messages' => [{}],
31
+ 'account' => {}
32
+ }
33
+ }
34
+
35
+ let(:content) {
36
+ {
37
+ 'thing' => 'value',
38
+ '_links' => links,
39
+ '_forms' => forms,
40
+ '_embedded' => embedded
41
+ }
42
+ }
43
+
44
+ before do
45
+ initialize_logger
46
+ end
47
+
48
+ it "initializes with content" do
49
+ resource = Mode::API::Resource.new(content)
50
+
51
+ resource.thing.should == 'value'
52
+
53
+ expect { resource.thing2 }.to raise_error
54
+
55
+ link = resource.links('self')
56
+ link.href.should == links['self']['href']
57
+
58
+ form = resource.forms('create')
59
+ form.action.should == forms['create']['action']
60
+
61
+ messages = resource.embedded('messages')
62
+ messages.length.should == 1
63
+ messages.first.should be_instance_of(Mode::API::Resource)
64
+ messages.first.content.should == embedded['messages'].first
65
+
66
+ account = resource.embedded('account')
67
+ account.should be_instance_of(Mode::API::Resource)
68
+ account.content.should == embedded['account']
69
+ end
70
+ end
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Auth::AccessToken do
4
+ let(:resource) {
5
+ Mode::API::Resource.new({
6
+ 'name' => 'Personal',
7
+ 'token' => 'token123',
8
+ 'account_name' => 'besquared'
9
+ })
10
+ }
11
+
12
+ before do
13
+ initialize_logger
14
+ end
15
+
16
+ it "initializes with resource" do
17
+ token = Mode::Auth::AccessToken.new(resource)
18
+ token.name.should == 'Personal'
19
+ token.token.should == 'token123'
20
+ token.account_name.should == 'besquared'
21
+ end
22
+ end
@@ -0,0 +1,26 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Commands::AnalyzeField do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+
6
+ before do
7
+ initialize_logger
8
+ end
9
+
10
+ it "should analyze a csv" do
11
+ path = fixture_path('espn_draft.csv')
12
+ command = Mode::Commands::AnalyzeField.new(path, 1)
13
+
14
+ command.stub(:puts).and_return(true)
15
+
16
+ command.execute
17
+ end
18
+
19
+ it "should error out if the csv file doesn't exist" do
20
+ command = Mode::Commands::AnalyzeField.new("fake.yml", 1)
21
+
22
+ command.should_receive(:puts)
23
+
24
+ command.execute
25
+ end
26
+ end
@@ -1,15 +1,17 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe Mode::Commands::AnalyzeSchema do
4
- before(:each) do
5
- @tmpdir = Dir.mktmpdir
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+
6
+ before do
7
+ initialize_logger
6
8
  end
7
9
 
8
10
  it "should analyze a csv" do
9
- path = data_path('espn_draft.csv')
11
+ path = fixture_path('espn_draft.csv')
10
12
  command = Mode::Commands::AnalyzeSchema.new(path)
11
13
 
12
- command.stub(:say).and_return(true)
14
+ command.stub(:puts).and_return(true)
13
15
 
14
16
  command.execute
15
17
  end
@@ -17,7 +19,7 @@ describe Mode::Commands::AnalyzeSchema do
17
19
  it "should error out if the csv file doesn't exist" do
18
20
  command = Mode::Commands::AnalyzeSchema.new("fake.yml", keys: [0,1])
19
21
 
20
- command.should_not_receive(:say)
22
+ command.should_receive(:puts)
21
23
 
22
24
  command.execute
23
25
  end
@@ -0,0 +1,80 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Commands::Connect do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+ let(:config) { Mode::Connector::Config.init(tmpdir) }
6
+
7
+ before do
8
+ initialize_logger
9
+ end
10
+
11
+ it "initializes with command and connect config" do
12
+ connect = Mode::Commands::Connect.new('start')
13
+
14
+ connect.command.should == 'start'
15
+ connect.concurrency.should == 4
16
+ connect.configuration.should_not == nil
17
+ end
18
+
19
+ it "registers connector" do
20
+ connect = Mode::Commands::Connect.new('start')
21
+
22
+ registrar = double(:registrar, :perform! => true)
23
+ Mode::Connector::Registrar.should_receive(:new).and_return(registrar)
24
+
25
+ connect.send(:register!)
26
+ end
27
+
28
+ it "starts the daemon" do
29
+ connect = Mode::Commands::Connect.new('start')
30
+
31
+ Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
32
+
33
+ connect.send(:spawn_command!, 'start')
34
+ end
35
+
36
+ it 'restarts the daemon' do
37
+ connect = Mode::Commands::Connect.new('start')
38
+
39
+ Mode::Connector::Daemon.should_receive(:spawn!).twice.and_return(true)
40
+
41
+ connect.send(:spawn_restart!)
42
+ end
43
+
44
+ it 'spawns with the start command' do
45
+ connect = Mode::Commands::Connect.new('start')
46
+ Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
47
+
48
+ connect.send(:spawn!)
49
+ end
50
+
51
+ it 'spawns with the restart command' do
52
+ connect = Mode::Commands::Connect.new('restart')
53
+ Mode::Connector::Daemon.should_receive(:spawn!).twice.and_return(true)
54
+
55
+ connect.send(:spawn!)
56
+ end
57
+
58
+ it 'spawns with the stop command' do
59
+ connect = Mode::Commands::Connect.new('stop')
60
+ Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
61
+
62
+ connect.send(:spawn!)
63
+ end
64
+
65
+ it 'raises on spawn if the command is unrecognized' do
66
+ connect = Mode::Commands::Connect.new('fake')
67
+ Mode::Connector::Daemon.should_not_receive(:spawn!)
68
+
69
+ expect { connect.send(:spawn!) }.to raise_error
70
+ end
71
+
72
+ it 'executes the connector' do
73
+ connect = Mode::Commands::Connect.new('start')
74
+
75
+ connect.should_receive(:register!)
76
+ connect.should_receive(:spawn!)
77
+
78
+ connect.execute
79
+ end
80
+ end
@@ -0,0 +1,69 @@
1
+ require 'spec_helper'
2
+
3
+ class CommandClass
4
+ include Mode::Commands::Helpers
5
+ end
6
+
7
+ describe Mode::Commands::Helpers do
8
+ let(:tmpdir) { Dir.mktmpdir }
9
+ let(:command) { CommandClass.new }
10
+
11
+ before do
12
+ initialize_logger
13
+ end
14
+
15
+ it "provides a config dir" do
16
+ command.config_dir.should == File.expand_path("~/.mode/")
17
+ end
18
+
19
+ it "provides a config path" do
20
+ command.config_path.should == Mode::Config.full_path("~/.mode/")
21
+ end
22
+
23
+ it "provides config validation" do
24
+ command.stub(:config_dir).and_return(tmpdir)
25
+
26
+ expect { command.require_config! }.to raise_error
27
+
28
+ config = Mode::Config.init(tmpdir)
29
+
30
+ command.require_config!
31
+ end
32
+
33
+ it "provides credential validation" do
34
+ command.stub(:config_dir).and_return(tmpdir)
35
+
36
+ # raise for no config
37
+ expect { command.require_credentials! }.to raise_error
38
+
39
+ config = Mode::Config.init(tmpdir)
40
+
41
+ # raise for credentials not found
42
+ expect { command.require_credentials! }.to raise_error
43
+
44
+ config.username = 'besquared'
45
+ config.access_token = 'access_token'
46
+ config.save
47
+
48
+ command.require_credentials!
49
+ end
50
+
51
+ it "provides api request configuration" do
52
+ command.configure_api_requests!
53
+ Mode::API::Request.valid?.should == true
54
+ end
55
+
56
+ it "provides connect config validation" do
57
+ command.stub(:config_dir).and_return(tmpdir)
58
+
59
+ # raise for no config
60
+ expect { command.require_connect_config! }.to raise_error
61
+
62
+ config = Mode::Connector::Config.init(tmpdir)
63
+
64
+ config.data_sources << Mode::Connector::DataSource.new('test', {})
65
+ config.save
66
+
67
+ command.require_connect_config!
68
+ end
69
+ end
@@ -0,0 +1,155 @@
1
+ require 'spec_helper'
2
+
3
+ describe Mode::Commands::Import do
4
+ let(:tmpdir) { Dir.mktmpdir }
5
+ let(:packages_path) { Mode::API::Request.packages_path }
6
+
7
+ before do
8
+ initialize_logger
9
+ end
10
+
11
+ it "generates target resource path with a csv extension" do
12
+ source = fixture_path("espn_draft/data.csv")
13
+ path = Mode::Commands::Import.send(:target_resource_path, source)
14
+
15
+ path.should == "data/index.csv"
16
+ end
17
+
18
+ it "generates target resource path with a non-csv extension" do
19
+ source = fixture_path("espn_draft.json")
20
+ path = Mode::Commands::Import.send(:target_resource_path, source)
21
+
22
+ path.should == "data/index.json"
23
+ end
24
+
25
+ describe 'finding a package from a source file' do
26
+ it 'returns the package if the file is adjacent to datapackage.json' do
27
+ source = fixture_path("espn_draft/data.csv")
28
+ package = Mode::Commands::Import.find_package(source)
29
+
30
+ package.base_path.should == fixture_path("espn_draft")
31
+ end
32
+
33
+ it 'returns the package if the file is inside of the data subdirectory' do
34
+ source = fixture_path("country-codes/data/country-codes.csv")
35
+ package = Mode::Commands::Import.find_package(source)
36
+
37
+ package.base_path.should == fixture_path("country-codes")
38
+ end
39
+
40
+ it 'returns nil if a package is not found' do
41
+ source = fixture_path("espn_draft.csv")
42
+ Mode::Commands::Import.find_package(source).should == nil
43
+ end
44
+ end
45
+
46
+ describe 'finding a package from a source directory' do
47
+ it 'returns the package if it exists in the directory' do
48
+ source = fixture_path("espn_draft")
49
+ package = Mode::Commands::Import.find_package(source)
50
+
51
+ package.base_path.should == fixture_path("espn_draft")
52
+ end
53
+
54
+ it 'returns nil if the package is not found in the directory' do
55
+ source = fixture_path
56
+
57
+ Mode::Commands::Import.find_package(source).should == nil
58
+ end
59
+ end
60
+
61
+ describe 'building a package from a source file' do
62
+ it 'build the package' do
63
+ source = fixture_path("espn_draft.csv")
64
+ package = Mode::Commands::Import.build_package(source, :dest => tmpdir)
65
+
66
+ package.base_path.should == tmpdir
67
+ end
68
+
69
+ it 'returns nil if the file does not exist' do
70
+ source = fixture_path('nope')
71
+ Mode::Commands::Import.build_package(source).should == nil
72
+ end
73
+ end
74
+
75
+ describe 'finding or building a package from a source file' do
76
+ it 'returns the package and resource name for a file in a datapackage' do
77
+ source = fixture_path("espn_draft/data.csv")
78
+ package = Mode::Commands::Import.find_or_build_package(source)
79
+
80
+ package.resources.first.path.should == 'data.csv'
81
+ package.base_path.should == fixture_path("espn_draft")
82
+ end
83
+
84
+ it 'returns the package and resource name for a file outisde a datapackage' do
85
+ source = fixture_path("espn_draft.csv")
86
+ package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
87
+
88
+ package.base_path.should == tmpdir
89
+ package.resources.first.path.should == 'data/index.csv'
90
+ end
91
+ end
92
+
93
+ describe 'finding or building a package from a source directory' do
94
+ it 'returns the package and resource name' do
95
+ source = fixture_path("espn_draft")
96
+ package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
97
+
98
+ package.resources.first.path.should == 'data.csv'
99
+ package.base_path.should == fixture_path("espn_draft")
100
+ end
101
+ end
102
+
103
+ describe 'uploading a package' do
104
+ it 'uploads a package' do
105
+ Mode::API::Request.should_receive(:post).and_return(true)
106
+
107
+ source = fixture_path("espn_draft.csv")
108
+ package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
109
+
110
+ import = Mode::Commands::Import.new(source, 'besquared', 'espn_draft')
111
+
112
+ import.upload_package(package).should == true
113
+ end
114
+ end
115
+
116
+ describe 'executing an import' do
117
+ it 'finds and upload a package' do
118
+ Mode::API::Request.should_receive(:post).with(packages_path, :package => {
119
+ :contents => {
120
+ 'datapackage.json' => an_instance_of(String),
121
+ 'data/index.csv' => an_instance_of(Faraday::UploadIO)
122
+ }, :name => 'espn_draft'
123
+ }).and_return(:package)
124
+
125
+ source = fixture_path("espn_draft/data.csv")
126
+ import = Mode::Commands::Import.new(source, 'besquared', 'espn_draft')
127
+
128
+ resource = double(:resource)
129
+ import_params = { 'table_name' => 'espn_draft' }
130
+ import.should_receive(:parse_uploaded_response).with(:package).and_return(resource)
131
+ import.should_receive(:submit_import_form).with(resource, import_params).and_return(true)
132
+
133
+ import.execute
134
+ end
135
+
136
+ it 'finds and uploads country codes package' do
137
+ Mode::API::Request.should_receive(:post).with(packages_path, :package => {
138
+ :contents => {
139
+ 'datapackage.json' => an_instance_of(String),
140
+ 'data/index.csv' => an_instance_of(Faraday::UploadIO)
141
+ }, :name => 'country_codes'
142
+ }).and_return(:package)
143
+
144
+ source = fixture_path("country-codes/data/country-codes.csv")
145
+ import = Mode::Commands::Import.new(source, 'besquared', 'country_codes')
146
+
147
+ resource = double(:resource)
148
+ import_params = { 'table_name' => 'country_codes' }
149
+ import.should_receive(:parse_uploaded_response).with(:package).and_return(resource)
150
+ import.should_receive(:submit_import_form).with(resource, import_params).and_return(true)
151
+
152
+ import.execute
153
+ end
154
+ end
155
+ end