mode 0.0.5 → 0.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -1
- data/README.md +17 -22
- data/bin/mode +1 -1
- data/lib/mode.rb +34 -6
- data/lib/mode/api/form.rb +53 -0
- data/lib/mode/api/link.rb +31 -0
- data/lib/mode/api/request.rb +181 -0
- data/lib/mode/api/resource.rb +67 -0
- data/lib/mode/auth/access_token.rb +23 -0
- data/lib/mode/cli.rb +3 -3
- data/lib/mode/cli/analyze.rb +1 -1
- data/lib/mode/cli/base.rb +5 -0
- data/lib/mode/cli/connect.rb +18 -0
- data/lib/mode/cli/helpers.rb +0 -9
- data/lib/mode/cli/import.rb +9 -38
- data/lib/mode/cli/login.rb +13 -0
- data/lib/mode/cli/package.rb +2 -5
- data/lib/mode/commands/analyze_field.rb +20 -21
- data/lib/mode/commands/analyze_schema.rb +69 -48
- data/lib/mode/commands/connect.rb +78 -0
- data/lib/mode/commands/helpers.rb +54 -0
- data/lib/mode/commands/import.rb +209 -20
- data/lib/mode/commands/login.rb +111 -0
- data/lib/mode/config.rb +13 -33
- data/lib/mode/configurable.rb +46 -0
- data/lib/mode/connector/config.rb +31 -0
- data/lib/mode/connector/daemon.rb +27 -0
- data/lib/mode/connector/data_source.rb +75 -0
- data/lib/mode/connector/dataset.rb +13 -0
- data/lib/mode/connector/message.rb +31 -0
- data/lib/mode/connector/poller.rb +27 -0
- data/lib/mode/connector/processor.rb +58 -0
- data/lib/mode/connector/registrar.rb +36 -0
- data/lib/mode/connector/scheduler.rb +62 -0
- data/lib/mode/connector/selector.rb +47 -0
- data/lib/mode/connector/type_map.rb +45 -0
- data/lib/mode/connector/uploader.rb +50 -0
- data/lib/mode/logger.rb +202 -0
- data/lib/mode/version.rb +1 -1
- data/mode.gemspec +13 -2
- data/spec/api/form_spec.rb +51 -0
- data/spec/api/link_spec.rb +23 -0
- data/spec/api/request_spec.rb +111 -0
- data/spec/api/resource_spec.rb +70 -0
- data/spec/auth/access_token_spec.rb +22 -0
- data/spec/commands/analyze_field_spec.rb +26 -0
- data/spec/commands/analyze_schema_spec.rb +7 -5
- data/spec/commands/connect_spec.rb +80 -0
- data/spec/commands/helpers_spec.rb +69 -0
- data/spec/commands/import_spec.rb +155 -0
- data/spec/commands/login_spec.rb +178 -0
- data/spec/config_spec.rb +9 -7
- data/spec/connector/config_spec.rb +46 -0
- data/spec/connector/daemon_spec.rb +30 -0
- data/spec/connector/data_source_spec.rb +73 -0
- data/spec/connector/message_spec.rb +22 -0
- data/spec/connector/poller_spec.rb +26 -0
- data/spec/connector/processor_spec.rb +93 -0
- data/spec/connector/registrar_spec.rb +53 -0
- data/spec/connector/scheduler_spec.rb +93 -0
- data/spec/connector/selector_spec.rb +54 -0
- data/spec/connector/type_map_spec.rb +45 -0
- data/spec/connector/uploader_spec.rb +55 -0
- data/spec/fixtures/country-codes/README.md +71 -0
- data/spec/fixtures/country-codes/data/country-codes.csv +250 -0
- data/spec/fixtures/country-codes/datapackage.json +142 -0
- data/spec/fixtures/country-codes/scripts/get_countries_of_earth.py +370 -0
- data/spec/fixtures/country-codes/scripts/reorder_columns.py +8 -0
- data/spec/fixtures/country-codes/scripts/requirements.pip +2 -0
- data/spec/fixtures/espn_draft.csv +473 -1
- data/spec/fixtures/espn_draft/data.csv +473 -0
- data/spec/fixtures/espn_draft/datapackage.json +43 -0
- data/spec/logger_spec.rb +79 -0
- data/spec/spec_helper.rb +6 -1
- metadata +156 -19
- data/lib/mode/cli/setup.rb +0 -12
- data/lib/mode/commands/package.rb +0 -56
- data/lib/mode/commands/setup.rb +0 -36
- data/lib/mode/package_builder.rb +0 -57
- data/spec/commands/setup_spec.rb +0 -62
- data/spec/fixtures/MOCK_DATA.csv +0 -100001
- data/spec/fixtures/cb_clean_small.csv +0 -100000
- data/spec/fixtures/duplicate_keys.csv +0 -3
- data/spec/fixtures/format_examples.csv.txt +0 -6
- data/spec/fixtures/format_examples_after_excel.csv.txt +0 -1
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Mode::API::Resource do
|
4
|
+
let(:links) {
|
5
|
+
{
|
6
|
+
'self' => {
|
7
|
+
'templated' => true,
|
8
|
+
'href' => '/path/to/thing'
|
9
|
+
}
|
10
|
+
}
|
11
|
+
}
|
12
|
+
|
13
|
+
let(:forms) {
|
14
|
+
{
|
15
|
+
'create' => {
|
16
|
+
'method' => 'post',
|
17
|
+
'action' => '/widgets',
|
18
|
+
'input' => {
|
19
|
+
'widgets' => {
|
20
|
+
'type' => 'text',
|
21
|
+
'value' => '500'
|
22
|
+
}
|
23
|
+
}
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
27
|
+
|
28
|
+
let(:embedded) {
|
29
|
+
{
|
30
|
+
'messages' => [{}],
|
31
|
+
'account' => {}
|
32
|
+
}
|
33
|
+
}
|
34
|
+
|
35
|
+
let(:content) {
|
36
|
+
{
|
37
|
+
'thing' => 'value',
|
38
|
+
'_links' => links,
|
39
|
+
'_forms' => forms,
|
40
|
+
'_embedded' => embedded
|
41
|
+
}
|
42
|
+
}
|
43
|
+
|
44
|
+
before do
|
45
|
+
initialize_logger
|
46
|
+
end
|
47
|
+
|
48
|
+
it "initializes with content" do
|
49
|
+
resource = Mode::API::Resource.new(content)
|
50
|
+
|
51
|
+
resource.thing.should == 'value'
|
52
|
+
|
53
|
+
expect { resource.thing2 }.to raise_error
|
54
|
+
|
55
|
+
link = resource.links('self')
|
56
|
+
link.href.should == links['self']['href']
|
57
|
+
|
58
|
+
form = resource.forms('create')
|
59
|
+
form.action.should == forms['create']['action']
|
60
|
+
|
61
|
+
messages = resource.embedded('messages')
|
62
|
+
messages.length.should == 1
|
63
|
+
messages.first.should be_instance_of(Mode::API::Resource)
|
64
|
+
messages.first.content.should == embedded['messages'].first
|
65
|
+
|
66
|
+
account = resource.embedded('account')
|
67
|
+
account.should be_instance_of(Mode::API::Resource)
|
68
|
+
account.content.should == embedded['account']
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Mode::Auth::AccessToken do
|
4
|
+
let(:resource) {
|
5
|
+
Mode::API::Resource.new({
|
6
|
+
'name' => 'Personal',
|
7
|
+
'token' => 'token123',
|
8
|
+
'account_name' => 'besquared'
|
9
|
+
})
|
10
|
+
}
|
11
|
+
|
12
|
+
before do
|
13
|
+
initialize_logger
|
14
|
+
end
|
15
|
+
|
16
|
+
it "initializes with resource" do
|
17
|
+
token = Mode::Auth::AccessToken.new(resource)
|
18
|
+
token.name.should == 'Personal'
|
19
|
+
token.token.should == 'token123'
|
20
|
+
token.account_name.should == 'besquared'
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Mode::Commands::AnalyzeField do
|
4
|
+
let(:tmpdir) { Dir.mktmpdir }
|
5
|
+
|
6
|
+
before do
|
7
|
+
initialize_logger
|
8
|
+
end
|
9
|
+
|
10
|
+
it "should analyze a csv" do
|
11
|
+
path = fixture_path('espn_draft.csv')
|
12
|
+
command = Mode::Commands::AnalyzeField.new(path, 1)
|
13
|
+
|
14
|
+
command.stub(:puts).and_return(true)
|
15
|
+
|
16
|
+
command.execute
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should error out if the csv file doesn't exist" do
|
20
|
+
command = Mode::Commands::AnalyzeField.new("fake.yml", 1)
|
21
|
+
|
22
|
+
command.should_receive(:puts)
|
23
|
+
|
24
|
+
command.execute
|
25
|
+
end
|
26
|
+
end
|
@@ -1,15 +1,17 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe Mode::Commands::AnalyzeSchema do
|
4
|
-
|
5
|
-
|
4
|
+
let(:tmpdir) { Dir.mktmpdir }
|
5
|
+
|
6
|
+
before do
|
7
|
+
initialize_logger
|
6
8
|
end
|
7
9
|
|
8
10
|
it "should analyze a csv" do
|
9
|
-
path =
|
11
|
+
path = fixture_path('espn_draft.csv')
|
10
12
|
command = Mode::Commands::AnalyzeSchema.new(path)
|
11
13
|
|
12
|
-
command.stub(:
|
14
|
+
command.stub(:puts).and_return(true)
|
13
15
|
|
14
16
|
command.execute
|
15
17
|
end
|
@@ -17,7 +19,7 @@ describe Mode::Commands::AnalyzeSchema do
|
|
17
19
|
it "should error out if the csv file doesn't exist" do
|
18
20
|
command = Mode::Commands::AnalyzeSchema.new("fake.yml", keys: [0,1])
|
19
21
|
|
20
|
-
command.
|
22
|
+
command.should_receive(:puts)
|
21
23
|
|
22
24
|
command.execute
|
23
25
|
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Mode::Commands::Connect do
|
4
|
+
let(:tmpdir) { Dir.mktmpdir }
|
5
|
+
let(:config) { Mode::Connector::Config.init(tmpdir) }
|
6
|
+
|
7
|
+
before do
|
8
|
+
initialize_logger
|
9
|
+
end
|
10
|
+
|
11
|
+
it "initializes with command and connect config" do
|
12
|
+
connect = Mode::Commands::Connect.new('start')
|
13
|
+
|
14
|
+
connect.command.should == 'start'
|
15
|
+
connect.concurrency.should == 4
|
16
|
+
connect.configuration.should_not == nil
|
17
|
+
end
|
18
|
+
|
19
|
+
it "registers connector" do
|
20
|
+
connect = Mode::Commands::Connect.new('start')
|
21
|
+
|
22
|
+
registrar = double(:registrar, :perform! => true)
|
23
|
+
Mode::Connector::Registrar.should_receive(:new).and_return(registrar)
|
24
|
+
|
25
|
+
connect.send(:register!)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "starts the daemon" do
|
29
|
+
connect = Mode::Commands::Connect.new('start')
|
30
|
+
|
31
|
+
Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
|
32
|
+
|
33
|
+
connect.send(:spawn_command!, 'start')
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'restarts the daemon' do
|
37
|
+
connect = Mode::Commands::Connect.new('start')
|
38
|
+
|
39
|
+
Mode::Connector::Daemon.should_receive(:spawn!).twice.and_return(true)
|
40
|
+
|
41
|
+
connect.send(:spawn_restart!)
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'spawns with the start command' do
|
45
|
+
connect = Mode::Commands::Connect.new('start')
|
46
|
+
Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
|
47
|
+
|
48
|
+
connect.send(:spawn!)
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'spawns with the restart command' do
|
52
|
+
connect = Mode::Commands::Connect.new('restart')
|
53
|
+
Mode::Connector::Daemon.should_receive(:spawn!).twice.and_return(true)
|
54
|
+
|
55
|
+
connect.send(:spawn!)
|
56
|
+
end
|
57
|
+
|
58
|
+
it 'spawns with the stop command' do
|
59
|
+
connect = Mode::Commands::Connect.new('stop')
|
60
|
+
Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
|
61
|
+
|
62
|
+
connect.send(:spawn!)
|
63
|
+
end
|
64
|
+
|
65
|
+
it 'raises on spawn if the command is unrecognized' do
|
66
|
+
connect = Mode::Commands::Connect.new('fake')
|
67
|
+
Mode::Connector::Daemon.should_not_receive(:spawn!)
|
68
|
+
|
69
|
+
expect { connect.send(:spawn!) }.to raise_error
|
70
|
+
end
|
71
|
+
|
72
|
+
it 'executes the connector' do
|
73
|
+
connect = Mode::Commands::Connect.new('start')
|
74
|
+
|
75
|
+
connect.should_receive(:register!)
|
76
|
+
connect.should_receive(:spawn!)
|
77
|
+
|
78
|
+
connect.execute
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
class CommandClass
|
4
|
+
include Mode::Commands::Helpers
|
5
|
+
end
|
6
|
+
|
7
|
+
describe Mode::Commands::Helpers do
|
8
|
+
let(:tmpdir) { Dir.mktmpdir }
|
9
|
+
let(:command) { CommandClass.new }
|
10
|
+
|
11
|
+
before do
|
12
|
+
initialize_logger
|
13
|
+
end
|
14
|
+
|
15
|
+
it "provides a config dir" do
|
16
|
+
command.config_dir.should == File.expand_path("~/.mode/")
|
17
|
+
end
|
18
|
+
|
19
|
+
it "provides a config path" do
|
20
|
+
command.config_path.should == Mode::Config.full_path("~/.mode/")
|
21
|
+
end
|
22
|
+
|
23
|
+
it "provides config validation" do
|
24
|
+
command.stub(:config_dir).and_return(tmpdir)
|
25
|
+
|
26
|
+
expect { command.require_config! }.to raise_error
|
27
|
+
|
28
|
+
config = Mode::Config.init(tmpdir)
|
29
|
+
|
30
|
+
command.require_config!
|
31
|
+
end
|
32
|
+
|
33
|
+
it "provides credential validation" do
|
34
|
+
command.stub(:config_dir).and_return(tmpdir)
|
35
|
+
|
36
|
+
# raise for no config
|
37
|
+
expect { command.require_credentials! }.to raise_error
|
38
|
+
|
39
|
+
config = Mode::Config.init(tmpdir)
|
40
|
+
|
41
|
+
# raise for credentials not found
|
42
|
+
expect { command.require_credentials! }.to raise_error
|
43
|
+
|
44
|
+
config.username = 'besquared'
|
45
|
+
config.access_token = 'access_token'
|
46
|
+
config.save
|
47
|
+
|
48
|
+
command.require_credentials!
|
49
|
+
end
|
50
|
+
|
51
|
+
it "provides api request configuration" do
|
52
|
+
command.configure_api_requests!
|
53
|
+
Mode::API::Request.valid?.should == true
|
54
|
+
end
|
55
|
+
|
56
|
+
it "provides connect config validation" do
|
57
|
+
command.stub(:config_dir).and_return(tmpdir)
|
58
|
+
|
59
|
+
# raise for no config
|
60
|
+
expect { command.require_connect_config! }.to raise_error
|
61
|
+
|
62
|
+
config = Mode::Connector::Config.init(tmpdir)
|
63
|
+
|
64
|
+
config.data_sources << Mode::Connector::DataSource.new('test', {})
|
65
|
+
config.save
|
66
|
+
|
67
|
+
command.require_connect_config!
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,155 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Mode::Commands::Import do
|
4
|
+
let(:tmpdir) { Dir.mktmpdir }
|
5
|
+
let(:packages_path) { Mode::API::Request.packages_path }
|
6
|
+
|
7
|
+
before do
|
8
|
+
initialize_logger
|
9
|
+
end
|
10
|
+
|
11
|
+
it "generates target resource path with a csv extension" do
|
12
|
+
source = fixture_path("espn_draft/data.csv")
|
13
|
+
path = Mode::Commands::Import.send(:target_resource_path, source)
|
14
|
+
|
15
|
+
path.should == "data/index.csv"
|
16
|
+
end
|
17
|
+
|
18
|
+
it "generates target resource path with a non-csv extension" do
|
19
|
+
source = fixture_path("espn_draft.json")
|
20
|
+
path = Mode::Commands::Import.send(:target_resource_path, source)
|
21
|
+
|
22
|
+
path.should == "data/index.json"
|
23
|
+
end
|
24
|
+
|
25
|
+
describe 'finding a package from a source file' do
|
26
|
+
it 'returns the package if the file is adjacent to datapackage.json' do
|
27
|
+
source = fixture_path("espn_draft/data.csv")
|
28
|
+
package = Mode::Commands::Import.find_package(source)
|
29
|
+
|
30
|
+
package.base_path.should == fixture_path("espn_draft")
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'returns the package if the file is inside of the data subdirectory' do
|
34
|
+
source = fixture_path("country-codes/data/country-codes.csv")
|
35
|
+
package = Mode::Commands::Import.find_package(source)
|
36
|
+
|
37
|
+
package.base_path.should == fixture_path("country-codes")
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'returns nil if a package is not found' do
|
41
|
+
source = fixture_path("espn_draft.csv")
|
42
|
+
Mode::Commands::Import.find_package(source).should == nil
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
describe 'finding a package from a source directory' do
|
47
|
+
it 'returns the package if it exists in the directory' do
|
48
|
+
source = fixture_path("espn_draft")
|
49
|
+
package = Mode::Commands::Import.find_package(source)
|
50
|
+
|
51
|
+
package.base_path.should == fixture_path("espn_draft")
|
52
|
+
end
|
53
|
+
|
54
|
+
it 'returns nil if the package is not found in the directory' do
|
55
|
+
source = fixture_path
|
56
|
+
|
57
|
+
Mode::Commands::Import.find_package(source).should == nil
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
describe 'building a package from a source file' do
|
62
|
+
it 'build the package' do
|
63
|
+
source = fixture_path("espn_draft.csv")
|
64
|
+
package = Mode::Commands::Import.build_package(source, :dest => tmpdir)
|
65
|
+
|
66
|
+
package.base_path.should == tmpdir
|
67
|
+
end
|
68
|
+
|
69
|
+
it 'returns nil if the file does not exist' do
|
70
|
+
source = fixture_path('nope')
|
71
|
+
Mode::Commands::Import.build_package(source).should == nil
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe 'finding or building a package from a source file' do
|
76
|
+
it 'returns the package and resource name for a file in a datapackage' do
|
77
|
+
source = fixture_path("espn_draft/data.csv")
|
78
|
+
package = Mode::Commands::Import.find_or_build_package(source)
|
79
|
+
|
80
|
+
package.resources.first.path.should == 'data.csv'
|
81
|
+
package.base_path.should == fixture_path("espn_draft")
|
82
|
+
end
|
83
|
+
|
84
|
+
it 'returns the package and resource name for a file outisde a datapackage' do
|
85
|
+
source = fixture_path("espn_draft.csv")
|
86
|
+
package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
|
87
|
+
|
88
|
+
package.base_path.should == tmpdir
|
89
|
+
package.resources.first.path.should == 'data/index.csv'
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
describe 'finding or building a package from a source directory' do
|
94
|
+
it 'returns the package and resource name' do
|
95
|
+
source = fixture_path("espn_draft")
|
96
|
+
package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
|
97
|
+
|
98
|
+
package.resources.first.path.should == 'data.csv'
|
99
|
+
package.base_path.should == fixture_path("espn_draft")
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
describe 'uploading a package' do
|
104
|
+
it 'uploads a package' do
|
105
|
+
Mode::API::Request.should_receive(:post).and_return(true)
|
106
|
+
|
107
|
+
source = fixture_path("espn_draft.csv")
|
108
|
+
package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
|
109
|
+
|
110
|
+
import = Mode::Commands::Import.new(source, 'besquared', 'espn_draft')
|
111
|
+
|
112
|
+
import.upload_package(package).should == true
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
describe 'executing an import' do
|
117
|
+
it 'finds and upload a package' do
|
118
|
+
Mode::API::Request.should_receive(:post).with(packages_path, :package => {
|
119
|
+
:contents => {
|
120
|
+
'datapackage.json' => an_instance_of(String),
|
121
|
+
'data/index.csv' => an_instance_of(Faraday::UploadIO)
|
122
|
+
}, :name => 'espn_draft'
|
123
|
+
}).and_return(:package)
|
124
|
+
|
125
|
+
source = fixture_path("espn_draft/data.csv")
|
126
|
+
import = Mode::Commands::Import.new(source, 'besquared', 'espn_draft')
|
127
|
+
|
128
|
+
resource = double(:resource)
|
129
|
+
import_params = { 'table_name' => 'espn_draft' }
|
130
|
+
import.should_receive(:parse_uploaded_response).with(:package).and_return(resource)
|
131
|
+
import.should_receive(:submit_import_form).with(resource, import_params).and_return(true)
|
132
|
+
|
133
|
+
import.execute
|
134
|
+
end
|
135
|
+
|
136
|
+
it 'finds and uploads country codes package' do
|
137
|
+
Mode::API::Request.should_receive(:post).with(packages_path, :package => {
|
138
|
+
:contents => {
|
139
|
+
'datapackage.json' => an_instance_of(String),
|
140
|
+
'data/index.csv' => an_instance_of(Faraday::UploadIO)
|
141
|
+
}, :name => 'country_codes'
|
142
|
+
}).and_return(:package)
|
143
|
+
|
144
|
+
source = fixture_path("country-codes/data/country-codes.csv")
|
145
|
+
import = Mode::Commands::Import.new(source, 'besquared', 'country_codes')
|
146
|
+
|
147
|
+
resource = double(:resource)
|
148
|
+
import_params = { 'table_name' => 'country_codes' }
|
149
|
+
import.should_receive(:parse_uploaded_response).with(:package).and_return(resource)
|
150
|
+
import.should_receive(:submit_import_form).with(resource, import_params).and_return(true)
|
151
|
+
|
152
|
+
import.execute
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|