mode 0.0.5 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -1
- data/README.md +17 -22
- data/bin/mode +1 -1
- data/lib/mode.rb +34 -6
- data/lib/mode/api/form.rb +53 -0
- data/lib/mode/api/link.rb +31 -0
- data/lib/mode/api/request.rb +181 -0
- data/lib/mode/api/resource.rb +67 -0
- data/lib/mode/auth/access_token.rb +23 -0
- data/lib/mode/cli.rb +3 -3
- data/lib/mode/cli/analyze.rb +1 -1
- data/lib/mode/cli/base.rb +5 -0
- data/lib/mode/cli/connect.rb +18 -0
- data/lib/mode/cli/helpers.rb +0 -9
- data/lib/mode/cli/import.rb +9 -38
- data/lib/mode/cli/login.rb +13 -0
- data/lib/mode/cli/package.rb +2 -5
- data/lib/mode/commands/analyze_field.rb +20 -21
- data/lib/mode/commands/analyze_schema.rb +69 -48
- data/lib/mode/commands/connect.rb +78 -0
- data/lib/mode/commands/helpers.rb +54 -0
- data/lib/mode/commands/import.rb +209 -20
- data/lib/mode/commands/login.rb +111 -0
- data/lib/mode/config.rb +13 -33
- data/lib/mode/configurable.rb +46 -0
- data/lib/mode/connector/config.rb +31 -0
- data/lib/mode/connector/daemon.rb +27 -0
- data/lib/mode/connector/data_source.rb +75 -0
- data/lib/mode/connector/dataset.rb +13 -0
- data/lib/mode/connector/message.rb +31 -0
- data/lib/mode/connector/poller.rb +27 -0
- data/lib/mode/connector/processor.rb +58 -0
- data/lib/mode/connector/registrar.rb +36 -0
- data/lib/mode/connector/scheduler.rb +62 -0
- data/lib/mode/connector/selector.rb +47 -0
- data/lib/mode/connector/type_map.rb +45 -0
- data/lib/mode/connector/uploader.rb +50 -0
- data/lib/mode/logger.rb +202 -0
- data/lib/mode/version.rb +1 -1
- data/mode.gemspec +13 -2
- data/spec/api/form_spec.rb +51 -0
- data/spec/api/link_spec.rb +23 -0
- data/spec/api/request_spec.rb +111 -0
- data/spec/api/resource_spec.rb +70 -0
- data/spec/auth/access_token_spec.rb +22 -0
- data/spec/commands/analyze_field_spec.rb +26 -0
- data/spec/commands/analyze_schema_spec.rb +7 -5
- data/spec/commands/connect_spec.rb +80 -0
- data/spec/commands/helpers_spec.rb +69 -0
- data/spec/commands/import_spec.rb +155 -0
- data/spec/commands/login_spec.rb +178 -0
- data/spec/config_spec.rb +9 -7
- data/spec/connector/config_spec.rb +46 -0
- data/spec/connector/daemon_spec.rb +30 -0
- data/spec/connector/data_source_spec.rb +73 -0
- data/spec/connector/message_spec.rb +22 -0
- data/spec/connector/poller_spec.rb +26 -0
- data/spec/connector/processor_spec.rb +93 -0
- data/spec/connector/registrar_spec.rb +53 -0
- data/spec/connector/scheduler_spec.rb +93 -0
- data/spec/connector/selector_spec.rb +54 -0
- data/spec/connector/type_map_spec.rb +45 -0
- data/spec/connector/uploader_spec.rb +55 -0
- data/spec/fixtures/country-codes/README.md +71 -0
- data/spec/fixtures/country-codes/data/country-codes.csv +250 -0
- data/spec/fixtures/country-codes/datapackage.json +142 -0
- data/spec/fixtures/country-codes/scripts/get_countries_of_earth.py +370 -0
- data/spec/fixtures/country-codes/scripts/reorder_columns.py +8 -0
- data/spec/fixtures/country-codes/scripts/requirements.pip +2 -0
- data/spec/fixtures/espn_draft.csv +473 -1
- data/spec/fixtures/espn_draft/data.csv +473 -0
- data/spec/fixtures/espn_draft/datapackage.json +43 -0
- data/spec/logger_spec.rb +79 -0
- data/spec/spec_helper.rb +6 -1
- metadata +156 -19
- data/lib/mode/cli/setup.rb +0 -12
- data/lib/mode/commands/package.rb +0 -56
- data/lib/mode/commands/setup.rb +0 -36
- data/lib/mode/package_builder.rb +0 -57
- data/spec/commands/setup_spec.rb +0 -62
- data/spec/fixtures/MOCK_DATA.csv +0 -100001
- data/spec/fixtures/cb_clean_small.csv +0 -100000
- data/spec/fixtures/duplicate_keys.csv +0 -3
- data/spec/fixtures/format_examples.csv.txt +0 -6
- data/spec/fixtures/format_examples_after_excel.csv.txt +0 -1
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Mode::API::Resource do
|
4
|
+
let(:links) {
|
5
|
+
{
|
6
|
+
'self' => {
|
7
|
+
'templated' => true,
|
8
|
+
'href' => '/path/to/thing'
|
9
|
+
}
|
10
|
+
}
|
11
|
+
}
|
12
|
+
|
13
|
+
let(:forms) {
|
14
|
+
{
|
15
|
+
'create' => {
|
16
|
+
'method' => 'post',
|
17
|
+
'action' => '/widgets',
|
18
|
+
'input' => {
|
19
|
+
'widgets' => {
|
20
|
+
'type' => 'text',
|
21
|
+
'value' => '500'
|
22
|
+
}
|
23
|
+
}
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
27
|
+
|
28
|
+
let(:embedded) {
|
29
|
+
{
|
30
|
+
'messages' => [{}],
|
31
|
+
'account' => {}
|
32
|
+
}
|
33
|
+
}
|
34
|
+
|
35
|
+
let(:content) {
|
36
|
+
{
|
37
|
+
'thing' => 'value',
|
38
|
+
'_links' => links,
|
39
|
+
'_forms' => forms,
|
40
|
+
'_embedded' => embedded
|
41
|
+
}
|
42
|
+
}
|
43
|
+
|
44
|
+
before do
|
45
|
+
initialize_logger
|
46
|
+
end
|
47
|
+
|
48
|
+
it "initializes with content" do
|
49
|
+
resource = Mode::API::Resource.new(content)
|
50
|
+
|
51
|
+
resource.thing.should == 'value'
|
52
|
+
|
53
|
+
expect { resource.thing2 }.to raise_error
|
54
|
+
|
55
|
+
link = resource.links('self')
|
56
|
+
link.href.should == links['self']['href']
|
57
|
+
|
58
|
+
form = resource.forms('create')
|
59
|
+
form.action.should == forms['create']['action']
|
60
|
+
|
61
|
+
messages = resource.embedded('messages')
|
62
|
+
messages.length.should == 1
|
63
|
+
messages.first.should be_instance_of(Mode::API::Resource)
|
64
|
+
messages.first.content.should == embedded['messages'].first
|
65
|
+
|
66
|
+
account = resource.embedded('account')
|
67
|
+
account.should be_instance_of(Mode::API::Resource)
|
68
|
+
account.content.should == embedded['account']
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Mode::Auth::AccessToken do
|
4
|
+
let(:resource) {
|
5
|
+
Mode::API::Resource.new({
|
6
|
+
'name' => 'Personal',
|
7
|
+
'token' => 'token123',
|
8
|
+
'account_name' => 'besquared'
|
9
|
+
})
|
10
|
+
}
|
11
|
+
|
12
|
+
before do
|
13
|
+
initialize_logger
|
14
|
+
end
|
15
|
+
|
16
|
+
it "initializes with resource" do
|
17
|
+
token = Mode::Auth::AccessToken.new(resource)
|
18
|
+
token.name.should == 'Personal'
|
19
|
+
token.token.should == 'token123'
|
20
|
+
token.account_name.should == 'besquared'
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Mode::Commands::AnalyzeField do
|
4
|
+
let(:tmpdir) { Dir.mktmpdir }
|
5
|
+
|
6
|
+
before do
|
7
|
+
initialize_logger
|
8
|
+
end
|
9
|
+
|
10
|
+
it "should analyze a csv" do
|
11
|
+
path = fixture_path('espn_draft.csv')
|
12
|
+
command = Mode::Commands::AnalyzeField.new(path, 1)
|
13
|
+
|
14
|
+
command.stub(:puts).and_return(true)
|
15
|
+
|
16
|
+
command.execute
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should error out if the csv file doesn't exist" do
|
20
|
+
command = Mode::Commands::AnalyzeField.new("fake.yml", 1)
|
21
|
+
|
22
|
+
command.should_receive(:puts)
|
23
|
+
|
24
|
+
command.execute
|
25
|
+
end
|
26
|
+
end
|
@@ -1,15 +1,17 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe Mode::Commands::AnalyzeSchema do
|
4
|
-
|
5
|
-
|
4
|
+
let(:tmpdir) { Dir.mktmpdir }
|
5
|
+
|
6
|
+
before do
|
7
|
+
initialize_logger
|
6
8
|
end
|
7
9
|
|
8
10
|
it "should analyze a csv" do
|
9
|
-
path =
|
11
|
+
path = fixture_path('espn_draft.csv')
|
10
12
|
command = Mode::Commands::AnalyzeSchema.new(path)
|
11
13
|
|
12
|
-
command.stub(:
|
14
|
+
command.stub(:puts).and_return(true)
|
13
15
|
|
14
16
|
command.execute
|
15
17
|
end
|
@@ -17,7 +19,7 @@ describe Mode::Commands::AnalyzeSchema do
|
|
17
19
|
it "should error out if the csv file doesn't exist" do
|
18
20
|
command = Mode::Commands::AnalyzeSchema.new("fake.yml", keys: [0,1])
|
19
21
|
|
20
|
-
command.
|
22
|
+
command.should_receive(:puts)
|
21
23
|
|
22
24
|
command.execute
|
23
25
|
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Mode::Commands::Connect do
|
4
|
+
let(:tmpdir) { Dir.mktmpdir }
|
5
|
+
let(:config) { Mode::Connector::Config.init(tmpdir) }
|
6
|
+
|
7
|
+
before do
|
8
|
+
initialize_logger
|
9
|
+
end
|
10
|
+
|
11
|
+
it "initializes with command and connect config" do
|
12
|
+
connect = Mode::Commands::Connect.new('start')
|
13
|
+
|
14
|
+
connect.command.should == 'start'
|
15
|
+
connect.concurrency.should == 4
|
16
|
+
connect.configuration.should_not == nil
|
17
|
+
end
|
18
|
+
|
19
|
+
it "registers connector" do
|
20
|
+
connect = Mode::Commands::Connect.new('start')
|
21
|
+
|
22
|
+
registrar = double(:registrar, :perform! => true)
|
23
|
+
Mode::Connector::Registrar.should_receive(:new).and_return(registrar)
|
24
|
+
|
25
|
+
connect.send(:register!)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "starts the daemon" do
|
29
|
+
connect = Mode::Commands::Connect.new('start')
|
30
|
+
|
31
|
+
Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
|
32
|
+
|
33
|
+
connect.send(:spawn_command!, 'start')
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'restarts the daemon' do
|
37
|
+
connect = Mode::Commands::Connect.new('start')
|
38
|
+
|
39
|
+
Mode::Connector::Daemon.should_receive(:spawn!).twice.and_return(true)
|
40
|
+
|
41
|
+
connect.send(:spawn_restart!)
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'spawns with the start command' do
|
45
|
+
connect = Mode::Commands::Connect.new('start')
|
46
|
+
Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
|
47
|
+
|
48
|
+
connect.send(:spawn!)
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'spawns with the restart command' do
|
52
|
+
connect = Mode::Commands::Connect.new('restart')
|
53
|
+
Mode::Connector::Daemon.should_receive(:spawn!).twice.and_return(true)
|
54
|
+
|
55
|
+
connect.send(:spawn!)
|
56
|
+
end
|
57
|
+
|
58
|
+
it 'spawns with the stop command' do
|
59
|
+
connect = Mode::Commands::Connect.new('stop')
|
60
|
+
Mode::Connector::Daemon.should_receive(:spawn!).and_return(true)
|
61
|
+
|
62
|
+
connect.send(:spawn!)
|
63
|
+
end
|
64
|
+
|
65
|
+
it 'raises on spawn if the command is unrecognized' do
|
66
|
+
connect = Mode::Commands::Connect.new('fake')
|
67
|
+
Mode::Connector::Daemon.should_not_receive(:spawn!)
|
68
|
+
|
69
|
+
expect { connect.send(:spawn!) }.to raise_error
|
70
|
+
end
|
71
|
+
|
72
|
+
it 'executes the connector' do
|
73
|
+
connect = Mode::Commands::Connect.new('start')
|
74
|
+
|
75
|
+
connect.should_receive(:register!)
|
76
|
+
connect.should_receive(:spawn!)
|
77
|
+
|
78
|
+
connect.execute
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
class CommandClass
|
4
|
+
include Mode::Commands::Helpers
|
5
|
+
end
|
6
|
+
|
7
|
+
describe Mode::Commands::Helpers do
|
8
|
+
let(:tmpdir) { Dir.mktmpdir }
|
9
|
+
let(:command) { CommandClass.new }
|
10
|
+
|
11
|
+
before do
|
12
|
+
initialize_logger
|
13
|
+
end
|
14
|
+
|
15
|
+
it "provides a config dir" do
|
16
|
+
command.config_dir.should == File.expand_path("~/.mode/")
|
17
|
+
end
|
18
|
+
|
19
|
+
it "provides a config path" do
|
20
|
+
command.config_path.should == Mode::Config.full_path("~/.mode/")
|
21
|
+
end
|
22
|
+
|
23
|
+
it "provides config validation" do
|
24
|
+
command.stub(:config_dir).and_return(tmpdir)
|
25
|
+
|
26
|
+
expect { command.require_config! }.to raise_error
|
27
|
+
|
28
|
+
config = Mode::Config.init(tmpdir)
|
29
|
+
|
30
|
+
command.require_config!
|
31
|
+
end
|
32
|
+
|
33
|
+
it "provides credential validation" do
|
34
|
+
command.stub(:config_dir).and_return(tmpdir)
|
35
|
+
|
36
|
+
# raise for no config
|
37
|
+
expect { command.require_credentials! }.to raise_error
|
38
|
+
|
39
|
+
config = Mode::Config.init(tmpdir)
|
40
|
+
|
41
|
+
# raise for credentials not found
|
42
|
+
expect { command.require_credentials! }.to raise_error
|
43
|
+
|
44
|
+
config.username = 'besquared'
|
45
|
+
config.access_token = 'access_token'
|
46
|
+
config.save
|
47
|
+
|
48
|
+
command.require_credentials!
|
49
|
+
end
|
50
|
+
|
51
|
+
it "provides api request configuration" do
|
52
|
+
command.configure_api_requests!
|
53
|
+
Mode::API::Request.valid?.should == true
|
54
|
+
end
|
55
|
+
|
56
|
+
it "provides connect config validation" do
|
57
|
+
command.stub(:config_dir).and_return(tmpdir)
|
58
|
+
|
59
|
+
# raise for no config
|
60
|
+
expect { command.require_connect_config! }.to raise_error
|
61
|
+
|
62
|
+
config = Mode::Connector::Config.init(tmpdir)
|
63
|
+
|
64
|
+
config.data_sources << Mode::Connector::DataSource.new('test', {})
|
65
|
+
config.save
|
66
|
+
|
67
|
+
command.require_connect_config!
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,155 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Mode::Commands::Import do
|
4
|
+
let(:tmpdir) { Dir.mktmpdir }
|
5
|
+
let(:packages_path) { Mode::API::Request.packages_path }
|
6
|
+
|
7
|
+
before do
|
8
|
+
initialize_logger
|
9
|
+
end
|
10
|
+
|
11
|
+
it "generates target resource path with a csv extension" do
|
12
|
+
source = fixture_path("espn_draft/data.csv")
|
13
|
+
path = Mode::Commands::Import.send(:target_resource_path, source)
|
14
|
+
|
15
|
+
path.should == "data/index.csv"
|
16
|
+
end
|
17
|
+
|
18
|
+
it "generates target resource path with a non-csv extension" do
|
19
|
+
source = fixture_path("espn_draft.json")
|
20
|
+
path = Mode::Commands::Import.send(:target_resource_path, source)
|
21
|
+
|
22
|
+
path.should == "data/index.json"
|
23
|
+
end
|
24
|
+
|
25
|
+
describe 'finding a package from a source file' do
|
26
|
+
it 'returns the package if the file is adjacent to datapackage.json' do
|
27
|
+
source = fixture_path("espn_draft/data.csv")
|
28
|
+
package = Mode::Commands::Import.find_package(source)
|
29
|
+
|
30
|
+
package.base_path.should == fixture_path("espn_draft")
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'returns the package if the file is inside of the data subdirectory' do
|
34
|
+
source = fixture_path("country-codes/data/country-codes.csv")
|
35
|
+
package = Mode::Commands::Import.find_package(source)
|
36
|
+
|
37
|
+
package.base_path.should == fixture_path("country-codes")
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'returns nil if a package is not found' do
|
41
|
+
source = fixture_path("espn_draft.csv")
|
42
|
+
Mode::Commands::Import.find_package(source).should == nil
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
describe 'finding a package from a source directory' do
|
47
|
+
it 'returns the package if it exists in the directory' do
|
48
|
+
source = fixture_path("espn_draft")
|
49
|
+
package = Mode::Commands::Import.find_package(source)
|
50
|
+
|
51
|
+
package.base_path.should == fixture_path("espn_draft")
|
52
|
+
end
|
53
|
+
|
54
|
+
it 'returns nil if the package is not found in the directory' do
|
55
|
+
source = fixture_path
|
56
|
+
|
57
|
+
Mode::Commands::Import.find_package(source).should == nil
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
describe 'building a package from a source file' do
|
62
|
+
it 'build the package' do
|
63
|
+
source = fixture_path("espn_draft.csv")
|
64
|
+
package = Mode::Commands::Import.build_package(source, :dest => tmpdir)
|
65
|
+
|
66
|
+
package.base_path.should == tmpdir
|
67
|
+
end
|
68
|
+
|
69
|
+
it 'returns nil if the file does not exist' do
|
70
|
+
source = fixture_path('nope')
|
71
|
+
Mode::Commands::Import.build_package(source).should == nil
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe 'finding or building a package from a source file' do
|
76
|
+
it 'returns the package and resource name for a file in a datapackage' do
|
77
|
+
source = fixture_path("espn_draft/data.csv")
|
78
|
+
package = Mode::Commands::Import.find_or_build_package(source)
|
79
|
+
|
80
|
+
package.resources.first.path.should == 'data.csv'
|
81
|
+
package.base_path.should == fixture_path("espn_draft")
|
82
|
+
end
|
83
|
+
|
84
|
+
it 'returns the package and resource name for a file outisde a datapackage' do
|
85
|
+
source = fixture_path("espn_draft.csv")
|
86
|
+
package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
|
87
|
+
|
88
|
+
package.base_path.should == tmpdir
|
89
|
+
package.resources.first.path.should == 'data/index.csv'
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
describe 'finding or building a package from a source directory' do
|
94
|
+
it 'returns the package and resource name' do
|
95
|
+
source = fixture_path("espn_draft")
|
96
|
+
package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
|
97
|
+
|
98
|
+
package.resources.first.path.should == 'data.csv'
|
99
|
+
package.base_path.should == fixture_path("espn_draft")
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
describe 'uploading a package' do
|
104
|
+
it 'uploads a package' do
|
105
|
+
Mode::API::Request.should_receive(:post).and_return(true)
|
106
|
+
|
107
|
+
source = fixture_path("espn_draft.csv")
|
108
|
+
package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
|
109
|
+
|
110
|
+
import = Mode::Commands::Import.new(source, 'besquared', 'espn_draft')
|
111
|
+
|
112
|
+
import.upload_package(package).should == true
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
describe 'executing an import' do
|
117
|
+
it 'finds and upload a package' do
|
118
|
+
Mode::API::Request.should_receive(:post).with(packages_path, :package => {
|
119
|
+
:contents => {
|
120
|
+
'datapackage.json' => an_instance_of(String),
|
121
|
+
'data/index.csv' => an_instance_of(Faraday::UploadIO)
|
122
|
+
}, :name => 'espn_draft'
|
123
|
+
}).and_return(:package)
|
124
|
+
|
125
|
+
source = fixture_path("espn_draft/data.csv")
|
126
|
+
import = Mode::Commands::Import.new(source, 'besquared', 'espn_draft')
|
127
|
+
|
128
|
+
resource = double(:resource)
|
129
|
+
import_params = { 'table_name' => 'espn_draft' }
|
130
|
+
import.should_receive(:parse_uploaded_response).with(:package).and_return(resource)
|
131
|
+
import.should_receive(:submit_import_form).with(resource, import_params).and_return(true)
|
132
|
+
|
133
|
+
import.execute
|
134
|
+
end
|
135
|
+
|
136
|
+
it 'finds and uploads country codes package' do
|
137
|
+
Mode::API::Request.should_receive(:post).with(packages_path, :package => {
|
138
|
+
:contents => {
|
139
|
+
'datapackage.json' => an_instance_of(String),
|
140
|
+
'data/index.csv' => an_instance_of(Faraday::UploadIO)
|
141
|
+
}, :name => 'country_codes'
|
142
|
+
}).and_return(:package)
|
143
|
+
|
144
|
+
source = fixture_path("country-codes/data/country-codes.csv")
|
145
|
+
import = Mode::Commands::Import.new(source, 'besquared', 'country_codes')
|
146
|
+
|
147
|
+
resource = double(:resource)
|
148
|
+
import_params = { 'table_name' => 'country_codes' }
|
149
|
+
import.should_receive(:parse_uploaded_response).with(:package).and_return(resource)
|
150
|
+
import.should_receive(:submit_import_form).with(resource, import_params).and_return(true)
|
151
|
+
|
152
|
+
import.execute
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|