mode 0.0.5 → 0.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -1
- data/README.md +17 -22
- data/bin/mode +1 -1
- data/lib/mode.rb +34 -6
- data/lib/mode/api/form.rb +53 -0
- data/lib/mode/api/link.rb +31 -0
- data/lib/mode/api/request.rb +181 -0
- data/lib/mode/api/resource.rb +67 -0
- data/lib/mode/auth/access_token.rb +23 -0
- data/lib/mode/cli.rb +3 -3
- data/lib/mode/cli/analyze.rb +1 -1
- data/lib/mode/cli/base.rb +5 -0
- data/lib/mode/cli/connect.rb +18 -0
- data/lib/mode/cli/helpers.rb +0 -9
- data/lib/mode/cli/import.rb +9 -38
- data/lib/mode/cli/login.rb +13 -0
- data/lib/mode/cli/package.rb +2 -5
- data/lib/mode/commands/analyze_field.rb +20 -21
- data/lib/mode/commands/analyze_schema.rb +69 -48
- data/lib/mode/commands/connect.rb +78 -0
- data/lib/mode/commands/helpers.rb +54 -0
- data/lib/mode/commands/import.rb +209 -20
- data/lib/mode/commands/login.rb +111 -0
- data/lib/mode/config.rb +13 -33
- data/lib/mode/configurable.rb +46 -0
- data/lib/mode/connector/config.rb +31 -0
- data/lib/mode/connector/daemon.rb +27 -0
- data/lib/mode/connector/data_source.rb +75 -0
- data/lib/mode/connector/dataset.rb +13 -0
- data/lib/mode/connector/message.rb +31 -0
- data/lib/mode/connector/poller.rb +27 -0
- data/lib/mode/connector/processor.rb +58 -0
- data/lib/mode/connector/registrar.rb +36 -0
- data/lib/mode/connector/scheduler.rb +62 -0
- data/lib/mode/connector/selector.rb +47 -0
- data/lib/mode/connector/type_map.rb +45 -0
- data/lib/mode/connector/uploader.rb +50 -0
- data/lib/mode/logger.rb +202 -0
- data/lib/mode/version.rb +1 -1
- data/mode.gemspec +13 -2
- data/spec/api/form_spec.rb +51 -0
- data/spec/api/link_spec.rb +23 -0
- data/spec/api/request_spec.rb +111 -0
- data/spec/api/resource_spec.rb +70 -0
- data/spec/auth/access_token_spec.rb +22 -0
- data/spec/commands/analyze_field_spec.rb +26 -0
- data/spec/commands/analyze_schema_spec.rb +7 -5
- data/spec/commands/connect_spec.rb +80 -0
- data/spec/commands/helpers_spec.rb +69 -0
- data/spec/commands/import_spec.rb +155 -0
- data/spec/commands/login_spec.rb +178 -0
- data/spec/config_spec.rb +9 -7
- data/spec/connector/config_spec.rb +46 -0
- data/spec/connector/daemon_spec.rb +30 -0
- data/spec/connector/data_source_spec.rb +73 -0
- data/spec/connector/message_spec.rb +22 -0
- data/spec/connector/poller_spec.rb +26 -0
- data/spec/connector/processor_spec.rb +93 -0
- data/spec/connector/registrar_spec.rb +53 -0
- data/spec/connector/scheduler_spec.rb +93 -0
- data/spec/connector/selector_spec.rb +54 -0
- data/spec/connector/type_map_spec.rb +45 -0
- data/spec/connector/uploader_spec.rb +55 -0
- data/spec/fixtures/country-codes/README.md +71 -0
- data/spec/fixtures/country-codes/data/country-codes.csv +250 -0
- data/spec/fixtures/country-codes/datapackage.json +142 -0
- data/spec/fixtures/country-codes/scripts/get_countries_of_earth.py +370 -0
- data/spec/fixtures/country-codes/scripts/reorder_columns.py +8 -0
- data/spec/fixtures/country-codes/scripts/requirements.pip +2 -0
- data/spec/fixtures/espn_draft.csv +473 -1
- data/spec/fixtures/espn_draft/data.csv +473 -0
- data/spec/fixtures/espn_draft/datapackage.json +43 -0
- data/spec/logger_spec.rb +79 -0
- data/spec/spec_helper.rb +6 -1
- metadata +156 -19
- data/lib/mode/cli/setup.rb +0 -12
- data/lib/mode/commands/package.rb +0 -56
- data/lib/mode/commands/setup.rb +0 -36
- data/lib/mode/package_builder.rb +0 -57
- data/spec/commands/setup_spec.rb +0 -62
- data/spec/fixtures/MOCK_DATA.csv +0 -100001
- data/spec/fixtures/cb_clean_small.csv +0 -100000
- data/spec/fixtures/duplicate_keys.csv +0 -3
- data/spec/fixtures/format_examples.csv.txt +0 -6
- data/spec/fixtures/format_examples_after_excel.csv.txt +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 916fc74b44c1c5b65091492b449464196a27067c
|
4
|
+
data.tar.gz: 41bdef70c556a8b1807e5e9f8fb71bfce3a39ded
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: aba504acfea6ea6cd6052666abd9f826c44e1241ba738b523089bedd8e4e700f4d40e78ba1c5df9adf8d6dfcf0f03003564114cf29a79bc597d98fca31ecbf4e
|
7
|
+
data.tar.gz: 0fc687a9b19413e7c47339aaa6ddd215145a374f1cbb5d068b37c2c5b134891553c3ff69d1a8891c805f3c97f4594bedac2778374986e632b487db28d3666858
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -6,7 +6,18 @@ This package provides command line tools for managing datasets and connecting da
|
|
6
6
|
* Personal data warehouse connectivity
|
7
7
|
* Dataset formatting and importing (CSV)
|
8
8
|
|
9
|
-
|
9
|
+
|
10
|
+
## Install Mode
|
11
|
+
|
12
|
+
**Mode requires Ruby 1.9 or newer**
|
13
|
+
|
14
|
+
If you don't currently have Ruby 1.9 or aren't sure then follow the directions for installing it before continuing.
|
15
|
+
|
16
|
+
```
|
17
|
+
gem install mode
|
18
|
+
```
|
19
|
+
|
20
|
+
## Install Ruby 1.9+ if it's not currently installed.
|
10
21
|
|
11
22
|
This package requires at least Ruby 1.9 and Ruby 2.0 is recommended.
|
12
23
|
|
@@ -26,26 +37,18 @@ ruby -e "$(curl -fsSL https://raw.github.com/Homebrew/homebrew/go/install)"
|
|
26
37
|
|
27
38
|
```
|
28
39
|
brew update
|
29
|
-
brew install rbenv ruby-build
|
40
|
+
brew install rbenv ruby-build rbenv-gem-rehash
|
30
41
|
echo 'eval "$(rbenv init -)"' >> ~/.bash_profile
|
31
42
|
source ~/.bash_profile
|
32
43
|
```
|
33
44
|
|
34
|
-
3\. Install Ruby
|
45
|
+
3\. Install Ruby
|
35
46
|
|
36
47
|
Note: This usually takes several minutes
|
37
48
|
|
38
49
|
```
|
39
50
|
rbenv install 2.0.0-p353
|
40
51
|
rbenv global 2.0.0-p353
|
41
|
-
rbenv rehash
|
42
|
-
```
|
43
|
-
|
44
|
-
4\. Install the mode gem
|
45
|
-
|
46
|
-
```
|
47
|
-
gem install mode
|
48
|
-
rbenv rehash
|
49
52
|
```
|
50
53
|
|
51
54
|
### OSX Combined
|
@@ -55,29 +58,21 @@ For convenience you can just copy and paste all the lines at once into your term
|
|
55
58
|
```
|
56
59
|
ruby -e "$(curl -fsSL https://raw.github.com/Homebrew/homebrew/go/install)"
|
57
60
|
brew update
|
58
|
-
brew install rbenv ruby-build
|
61
|
+
brew install rbenv ruby-build rbenv-gem-rehash
|
59
62
|
echo 'eval "$(rbenv init -)"' >> ~/.bash_profile
|
60
63
|
source ~/.bash_profile
|
61
64
|
rbenv install 2.0.0-p353
|
62
65
|
rbenv global 2.0.0-p353
|
63
|
-
gem install mode
|
64
|
-
rbenv rehash
|
65
66
|
```
|
66
67
|
|
67
68
|
### Windows
|
68
69
|
|
69
70
|
To install a current version of Ruby on windows complete the 2 steps below.
|
70
71
|
|
71
|
-
1\. Install Ruby
|
72
|
+
1\. Install Ruby
|
72
73
|
|
73
74
|
You can download the latest ruby version from [RubyInstaller](http://rubyinstaller.org/downloads/).
|
74
75
|
|
75
|
-
2\. Install the mode gem
|
76
|
-
|
77
|
-
```
|
78
|
-
$ gem install mode
|
79
|
-
```
|
80
|
-
|
81
76
|
## Setup
|
82
77
|
|
83
78
|
### Init
|
@@ -85,7 +80,7 @@ $ gem install mode
|
|
85
80
|
Initializes a new configuration file at the specified path which holds API credentials and other information
|
86
81
|
|
87
82
|
```
|
88
|
-
$ mode
|
83
|
+
$ mode login
|
89
84
|
|
90
85
|
Initializing configuration at /Users/josh/.mode.yml
|
91
86
|
Mode username: besquared
|
data/bin/mode
CHANGED
data/lib/mode.rb
CHANGED
@@ -1,19 +1,47 @@
|
|
1
1
|
require 'thor'
|
2
|
+
require 'faraday'
|
2
3
|
require 'data_kit'
|
3
4
|
require 'data_package'
|
4
5
|
|
5
6
|
require 'mode/version'
|
6
7
|
|
7
|
-
# Config
|
8
|
+
# Config & Logging
|
9
|
+
require 'mode/configurable'
|
8
10
|
require 'mode/config'
|
11
|
+
require 'mode/logger'
|
12
|
+
|
13
|
+
# Auth
|
14
|
+
require 'mode/auth/access_token'
|
15
|
+
|
16
|
+
|
17
|
+
# API
|
18
|
+
require 'mode/api/link'
|
19
|
+
require 'mode/api/form'
|
20
|
+
require 'mode/api/resource'
|
21
|
+
require 'mode/api/request'
|
22
|
+
|
23
|
+
# Connector
|
24
|
+
require 'mode/connector/config'
|
25
|
+
require 'mode/connector/type_map'
|
26
|
+
require 'mode/connector/data_source'
|
27
|
+
require 'mode/connector/dataset'
|
28
|
+
|
29
|
+
require 'mode/connector/registrar'
|
30
|
+
require 'mode/connector/scheduler'
|
31
|
+
require 'mode/connector/poller'
|
32
|
+
require 'mode/connector/processor'
|
33
|
+
require 'mode/connector/selector'
|
34
|
+
require 'mode/connector/uploader'
|
35
|
+
|
36
|
+
require 'mode/connector/daemon'
|
37
|
+
require 'mode/connector/message'
|
38
|
+
|
9
39
|
|
10
40
|
# Commands
|
11
41
|
require 'mode/commands/helpers'
|
12
|
-
|
42
|
+
|
43
|
+
require 'mode/commands/login'
|
13
44
|
require 'mode/commands/import'
|
14
|
-
require 'mode/commands/
|
45
|
+
require 'mode/commands/connect'
|
15
46
|
require 'mode/commands/analyze_field'
|
16
47
|
require 'mode/commands/analyze_schema'
|
17
|
-
|
18
|
-
# Utilities
|
19
|
-
require 'mode/package_builder'
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Mode
|
2
|
+
module API
|
3
|
+
class Form
|
4
|
+
attr_reader :content
|
5
|
+
|
6
|
+
def initialize(content)
|
7
|
+
@content = content
|
8
|
+
end
|
9
|
+
|
10
|
+
def method
|
11
|
+
content['method']
|
12
|
+
end
|
13
|
+
|
14
|
+
def action
|
15
|
+
content['action']
|
16
|
+
end
|
17
|
+
|
18
|
+
def input(name)
|
19
|
+
content['input'][name]
|
20
|
+
end
|
21
|
+
|
22
|
+
def submit!(params = {})
|
23
|
+
inputs = content['input']
|
24
|
+
params = input_values(inputs, params)
|
25
|
+
Mode::API::Request.send(method.to_sym, action, params)
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def input_values(inputs, params = {})
|
31
|
+
merged = {}
|
32
|
+
return merged if inputs.nil?
|
33
|
+
|
34
|
+
inputs.each do |name, value|
|
35
|
+
if value.instance_of?(Hash)
|
36
|
+
if value.has_key?('value')
|
37
|
+
# we reached a leaf
|
38
|
+
merged[name] = params[name] || value['value']
|
39
|
+
else
|
40
|
+
# we're at an intermediate node
|
41
|
+
merged[name] = input_values(value, params[name])
|
42
|
+
end
|
43
|
+
else
|
44
|
+
# non-field node
|
45
|
+
# merged[name] = params[name]
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
merged
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'uri_template'
|
2
|
+
|
3
|
+
module Mode
|
4
|
+
module API
|
5
|
+
class Link
|
6
|
+
attr_reader :content
|
7
|
+
|
8
|
+
def initialize(content)
|
9
|
+
@content = content
|
10
|
+
end
|
11
|
+
|
12
|
+
def href
|
13
|
+
content['href']
|
14
|
+
end
|
15
|
+
|
16
|
+
def templated
|
17
|
+
content['templated']
|
18
|
+
end
|
19
|
+
|
20
|
+
def expand(params = {})
|
21
|
+
template.expand(params)
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def template
|
27
|
+
URITemplate.new(href)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,181 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require 'json'
|
3
|
+
require 'faraday'
|
4
|
+
|
5
|
+
module Mode
|
6
|
+
module API
|
7
|
+
class Request
|
8
|
+
attr_reader :method, :path, :params
|
9
|
+
|
10
|
+
def initialize(method, path, params = {})
|
11
|
+
@method = method
|
12
|
+
@path = path
|
13
|
+
@params = params
|
14
|
+
end
|
15
|
+
|
16
|
+
def perform
|
17
|
+
response = http.send(method.to_sym, path, params)
|
18
|
+
response.success? ? success(response) : error(response)
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def http
|
24
|
+
@http ||= self.class.http
|
25
|
+
end
|
26
|
+
|
27
|
+
def parsed_body(response)
|
28
|
+
JSON.parse(response.body)
|
29
|
+
end
|
30
|
+
|
31
|
+
def error(response)
|
32
|
+
response.tap do |response|
|
33
|
+
log_error(response)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def success(response)
|
38
|
+
log_success(response)
|
39
|
+
Mode::API::Resource.new(parsed_body(response))
|
40
|
+
end
|
41
|
+
|
42
|
+
def log_error(response)
|
43
|
+
log_response(:error, response)
|
44
|
+
end
|
45
|
+
|
46
|
+
def log_success(response)
|
47
|
+
log_response(:info, response)
|
48
|
+
end
|
49
|
+
|
50
|
+
def log_response(level, response)
|
51
|
+
Mode::Logger.instance.send(level,
|
52
|
+
"API::Request", "#{response.status}", ["#{method.upcase} #{path}"])
|
53
|
+
end
|
54
|
+
|
55
|
+
class << self
|
56
|
+
#
|
57
|
+
# Configuration
|
58
|
+
#
|
59
|
+
|
60
|
+
attr_reader :environment
|
61
|
+
attr_reader :credentials
|
62
|
+
|
63
|
+
def valid?
|
64
|
+
!environment.nil?
|
65
|
+
end
|
66
|
+
|
67
|
+
def validate!
|
68
|
+
unless valid?
|
69
|
+
raise "Request API not configured. Please provide an API environment."
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def username
|
74
|
+
validate!
|
75
|
+
@credentials[:username]
|
76
|
+
end
|
77
|
+
|
78
|
+
def access_token
|
79
|
+
validate!
|
80
|
+
@credentials[:access_token]
|
81
|
+
end
|
82
|
+
|
83
|
+
def configure(environment, options = {})
|
84
|
+
@environment = environment
|
85
|
+
@credentials = options[:credentials]
|
86
|
+
end
|
87
|
+
|
88
|
+
#
|
89
|
+
# Request Methods
|
90
|
+
#
|
91
|
+
|
92
|
+
def head(path)
|
93
|
+
new(:head, resolve_path(path)).perform
|
94
|
+
end
|
95
|
+
|
96
|
+
def get(path)
|
97
|
+
new(:get, resolve_path(path)).perform
|
98
|
+
end
|
99
|
+
|
100
|
+
def post(path, params = {})
|
101
|
+
new(:post, resolve_path(path), params).perform
|
102
|
+
end
|
103
|
+
|
104
|
+
def put(path, params = {})
|
105
|
+
new(:put, resolve_path(path), params).perform
|
106
|
+
end
|
107
|
+
|
108
|
+
def delete(path, params = {})
|
109
|
+
new(:delete, resolve_path(path), params).perform
|
110
|
+
end
|
111
|
+
|
112
|
+
def resolve_path(path)
|
113
|
+
path.is_a?(Symbol) ? send("#{path}_path".to_sym) : path
|
114
|
+
end
|
115
|
+
|
116
|
+
#
|
117
|
+
# Routes
|
118
|
+
#
|
119
|
+
|
120
|
+
def base_path
|
121
|
+
"/api/#{username}"
|
122
|
+
end
|
123
|
+
|
124
|
+
def access_tokens_path
|
125
|
+
"#{base_path}/access_tokens"
|
126
|
+
end
|
127
|
+
|
128
|
+
def packages_path
|
129
|
+
"#{base_path}/packages"
|
130
|
+
end
|
131
|
+
|
132
|
+
def data_source_connections_path
|
133
|
+
"#{base_path}/data_source_connections"
|
134
|
+
end
|
135
|
+
|
136
|
+
def data_source_connection_path
|
137
|
+
[data_source_connections_path, access_token].join('/').strip
|
138
|
+
end
|
139
|
+
|
140
|
+
def data_source_connection_messages_path
|
141
|
+
[data_source_connection_path, 'messages'].join('/').strip
|
142
|
+
end
|
143
|
+
|
144
|
+
#
|
145
|
+
# Connection
|
146
|
+
#
|
147
|
+
def http(options = {})
|
148
|
+
@_http ||= Faraday.new(:url => base_url) do |builder|
|
149
|
+
builder.request :multipart # adds small overhead to each request
|
150
|
+
builder.request :url_encoded
|
151
|
+
builder.adapter Faraday.default_adapter
|
152
|
+
builder.basic_auth(username, access_token)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
private
|
157
|
+
|
158
|
+
def base_url
|
159
|
+
@base_url ||= URI::HTTP.build(:host => env_host, :port => env_port)
|
160
|
+
end
|
161
|
+
|
162
|
+
def env_host
|
163
|
+
case environment.to_sym
|
164
|
+
when :test
|
165
|
+
"test"
|
166
|
+
when :development
|
167
|
+
"localhost"
|
168
|
+
when :staging
|
169
|
+
"staging.modeanalytics.com"
|
170
|
+
else
|
171
|
+
"www.modeanalytics.com"
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
def env_port
|
176
|
+
environment.to_sym == :development ? 3000 : 80
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module Mode
|
2
|
+
module API
|
3
|
+
class Resource
|
4
|
+
attr_reader :content
|
5
|
+
|
6
|
+
def initialize(content)
|
7
|
+
@content = content
|
8
|
+
end
|
9
|
+
|
10
|
+
def links(name)
|
11
|
+
if has_link?(name)
|
12
|
+
Mode::API::Link.new(content['_links'][name])
|
13
|
+
else
|
14
|
+
nil
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def has_link?(name)
|
19
|
+
links = content && content['_links']
|
20
|
+
links.is_a?(Hash) && links.has_key?(name)
|
21
|
+
end
|
22
|
+
|
23
|
+
def forms(name)
|
24
|
+
if has_form?(name)
|
25
|
+
Mode::API::Form.new(content['_forms'][name])
|
26
|
+
else
|
27
|
+
nil
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def has_form?(name)
|
32
|
+
forms = content && content['_forms']
|
33
|
+
forms.is_a?(Hash) && forms.has_key?(name)
|
34
|
+
end
|
35
|
+
|
36
|
+
def embedded(name)
|
37
|
+
return nil unless has_embedded?(name)
|
38
|
+
|
39
|
+
embed = content['_embedded'][name]
|
40
|
+
|
41
|
+
if embed.is_a?(Array)
|
42
|
+
embed.map do |entry|
|
43
|
+
Mode::API::Resource.new(entry)
|
44
|
+
end
|
45
|
+
elsif embed.is_a?(Hash)
|
46
|
+
Mode::API::Resource.new(embed)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def has_embedded?(name)
|
51
|
+
embed = content['_embedded']
|
52
|
+
return false if embed.nil?
|
53
|
+
return false unless embed.is_a?(Hash)
|
54
|
+
return false unless embed.has_key?(name)
|
55
|
+
true
|
56
|
+
end
|
57
|
+
|
58
|
+
def method_missing(sym, *args, &block)
|
59
|
+
if content.has_key?(sym.to_s)
|
60
|
+
content[sym.to_s]
|
61
|
+
else
|
62
|
+
super
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|