mode 0.0.18 → 0.0.19
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -4
- data/bin/mode +2 -0
- data/lib/connect.rb +1 -1
- data/lib/mode.rb +0 -5
- data/lib/mode/cli.rb +1 -5
- data/lib/mode/cli/base.rb +7 -4
- data/lib/mode/cli/connect.rb +3 -1
- data/lib/mode/commands/login.rb +3 -3
- data/lib/mode/connector/daemonizer.rb +12 -3
- data/lib/mode/connector/scheduler.rb +3 -3
- data/lib/mode/version.rb +1 -1
- data/mode.gemspec +7 -7
- data/releases/mode-ruby-0.0.18.jar +0 -0
- metadata +19 -31
- data/lib/mode/cli/analyze.rb +0 -20
- data/lib/mode/cli/helpers.rb +0 -13
- data/lib/mode/cli/import.rb +0 -33
- data/lib/mode/cli/package.rb +0 -13
- data/lib/mode/commands/analyze_field.rb +0 -60
- data/lib/mode/commands/analyze_schema.rb +0 -111
- data/lib/mode/commands/import.rb +0 -224
- data/spec/commands/analyze_field_spec.rb +0 -26
- data/spec/commands/analyze_schema_spec.rb +0 -26
- data/spec/commands/import_spec.rb +0 -160
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f79719aeee1b231678a845b20822da9d23adfa88
|
4
|
+
data.tar.gz: 923422856ba88ce2e6b4a0c0036848982f1615d5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4f26aa68bd1871d9d9483dd9a48cdd3cdc064f5cfa8e9bb8cd46c5d3eb63ac09fa5933f2bc3680c07fbd20b4817f3819ca10b771fe55d0aab7f74238f5ad3374
|
7
|
+
data.tar.gz: 8a2a26041037d95ba48d6ab6f9077d5960364f7f21aa7c2a6e6f7f57d03134f1bfe1f8f2ab5c94f7e72bbd7b954568f64d214f35e304e4bbd73aeffa9d763dbc
|
data/.gitignore
CHANGED
data/bin/mode
CHANGED
data/lib/connect.rb
CHANGED
data/lib/mode.rb
CHANGED
@@ -1,7 +1,5 @@
|
|
1
1
|
require 'thor'
|
2
2
|
require 'faraday'
|
3
|
-
require 'data_kit'
|
4
|
-
require 'data_package'
|
5
3
|
|
6
4
|
require 'mode/version'
|
7
5
|
|
@@ -58,7 +56,4 @@ require 'mode/connector/daemonizer'
|
|
58
56
|
require 'mode/commands/helpers'
|
59
57
|
|
60
58
|
require 'mode/commands/login'
|
61
|
-
require 'mode/commands/import'
|
62
59
|
require 'mode/commands/connect'
|
63
|
-
require 'mode/commands/analyze_field'
|
64
|
-
require 'mode/commands/analyze_schema'
|
data/lib/mode/cli.rb
CHANGED
data/lib/mode/cli/base.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'sinatra/base'
|
2
|
+
|
1
3
|
module Mode
|
2
4
|
module CLI
|
3
5
|
class Base < Thor
|
@@ -12,9 +14,10 @@ module Mode
|
|
12
14
|
say "Mode CLI Version #{Mode::VERSION}"
|
13
15
|
end
|
14
16
|
|
15
|
-
|
16
|
-
|
17
|
-
|
17
|
+
desc "serve", "Daemonize admin server"
|
18
|
+
def serve
|
19
|
+
Mode::Connector::Daemon.new(:max_jobs => 4).start
|
20
|
+
end
|
18
21
|
end
|
19
22
|
end
|
20
|
-
end
|
23
|
+
end
|
data/lib/mode/cli/connect.rb
CHANGED
data/lib/mode/commands/login.rb
CHANGED
@@ -46,7 +46,7 @@ module Mode
|
|
46
46
|
def configure_credentials
|
47
47
|
say "Enter your Mode credentials:"
|
48
48
|
username = ask "Username:"
|
49
|
-
password = ask "Password:"
|
49
|
+
password = ask "Password:"#, :echo => false
|
50
50
|
|
51
51
|
# go ahead and configure API
|
52
52
|
configure_api(username, password)
|
@@ -153,7 +153,7 @@ module Mode
|
|
153
153
|
end
|
154
154
|
end
|
155
155
|
|
156
|
-
def update_configuration(environment, username, access_token)
|
156
|
+
def update_configuration(environment, username, access_token)
|
157
157
|
config = find_or_create_config
|
158
158
|
|
159
159
|
config.username = username
|
@@ -179,4 +179,4 @@ module Mode
|
|
179
179
|
end
|
180
180
|
end
|
181
181
|
end
|
182
|
-
end
|
182
|
+
end
|
@@ -37,9 +37,14 @@ module Mode
|
|
37
37
|
|
38
38
|
def start
|
39
39
|
abort "Mode connector already running" if alive?
|
40
|
-
abort "Insufficient permissions to start Mode connector" unless File.executable?(executable)
|
41
40
|
|
42
|
-
if
|
41
|
+
if RUBY_PLATFORM == 'java' && jar_file
|
42
|
+
pid = Spoon.spawnp("java", "-jar", jar_file, "serve")
|
43
|
+
else
|
44
|
+
pid = Spoon.spawnp(executable, *args.collect(&:to_s))
|
45
|
+
end
|
46
|
+
|
47
|
+
if pid
|
43
48
|
create_pid(pid)
|
44
49
|
STDOUT.puts "Mode connector running with pid #{pid}"
|
45
50
|
else
|
@@ -91,6 +96,10 @@ module Mode
|
|
91
96
|
exit!
|
92
97
|
end
|
93
98
|
|
99
|
+
def jar_file
|
100
|
+
File.dirname(__FILE__).match(/\/([\w\d\-\.]+)!/).to_a.last
|
101
|
+
end
|
102
|
+
|
94
103
|
def default_pid_file
|
95
104
|
File.join(Mode::Config.default_dir, 'connect.pid')
|
96
105
|
end
|
@@ -104,4 +113,4 @@ module Mode
|
|
104
113
|
end
|
105
114
|
end
|
106
115
|
end
|
107
|
-
end
|
116
|
+
end
|
@@ -25,14 +25,14 @@ module Mode
|
|
25
25
|
|
26
26
|
scheduler.interval('15m') { register }
|
27
27
|
scheduler.interval('5s') { process_messages }
|
28
|
-
scheduler.join
|
28
|
+
scheduler.join # don't join if we're running from inside sinatra
|
29
29
|
end
|
30
30
|
|
31
31
|
def stop!
|
32
32
|
stopper = Thread.new {
|
33
33
|
scheduler.stop # Stop polling
|
34
34
|
}
|
35
|
-
|
35
|
+
|
36
36
|
stopper.join # wait for jobs to finish
|
37
37
|
end
|
38
38
|
|
@@ -73,4 +73,4 @@ module Mode
|
|
73
73
|
end
|
74
74
|
end
|
75
75
|
end
|
76
|
-
end
|
76
|
+
end
|
data/lib/mode/version.rb
CHANGED
data/mode.gemspec
CHANGED
@@ -22,14 +22,14 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.add_runtime_dependency "thor"
|
23
23
|
spec.add_runtime_dependency "terminal-table"
|
24
24
|
|
25
|
-
#
|
26
|
-
spec.add_runtime_dependency "
|
27
|
-
spec.add_runtime_dependency "data_package"
|
25
|
+
# Web UI
|
26
|
+
spec.add_runtime_dependency "sinatra"
|
28
27
|
|
29
28
|
# HTTP
|
30
|
-
spec.add_runtime_dependency "faraday"
|
31
|
-
spec.add_runtime_dependency "faraday_middleware"
|
32
29
|
spec.add_runtime_dependency "uri_template"
|
30
|
+
spec.add_runtime_dependency "faraday", '~> 0.8.9'
|
31
|
+
spec.add_runtime_dependency "multipart-post", '~> 1.2.0'
|
32
|
+
spec.add_runtime_dependency "faraday_middleware", '~> 0.9.0'
|
33
33
|
|
34
34
|
# Connectivity
|
35
35
|
spec.add_runtime_dependency "sequel"
|
@@ -37,7 +37,7 @@ Gem::Specification.new do |spec|
|
|
37
37
|
# Concurrency & Processes
|
38
38
|
spec.add_runtime_dependency "spoon"
|
39
39
|
spec.add_runtime_dependency "rufus-scheduler"
|
40
|
-
|
40
|
+
|
41
41
|
# Web Management
|
42
42
|
# spec.add_runtime_dependency "launchy"
|
43
43
|
|
@@ -49,4 +49,4 @@ Gem::Specification.new do |spec|
|
|
49
49
|
spec.add_development_dependency "simplecov"
|
50
50
|
spec.add_development_dependency "pry"
|
51
51
|
spec.add_development_dependency "warbler"
|
52
|
-
end
|
52
|
+
end
|
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mode
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.19
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mode Analytics
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-03-
|
11
|
+
date: 2014-03-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -39,7 +39,7 @@ dependencies:
|
|
39
39
|
prerelease: false
|
40
40
|
type: :runtime
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: sinatra
|
43
43
|
version_requirements: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - '>='
|
@@ -53,7 +53,7 @@ dependencies:
|
|
53
53
|
prerelease: false
|
54
54
|
type: :runtime
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: uri_template
|
57
57
|
version_requirements: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - '>='
|
@@ -70,42 +70,42 @@ dependencies:
|
|
70
70
|
name: faraday
|
71
71
|
version_requirements: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ~>
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
75
|
+
version: 0.8.9
|
76
76
|
requirement: !ruby/object:Gem::Requirement
|
77
77
|
requirements:
|
78
|
-
- -
|
78
|
+
- - ~>
|
79
79
|
- !ruby/object:Gem::Version
|
80
|
-
version:
|
80
|
+
version: 0.8.9
|
81
81
|
prerelease: false
|
82
82
|
type: :runtime
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
84
|
+
name: multipart-post
|
85
85
|
version_requirements: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- -
|
87
|
+
- - ~>
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version:
|
89
|
+
version: 1.2.0
|
90
90
|
requirement: !ruby/object:Gem::Requirement
|
91
91
|
requirements:
|
92
|
-
- -
|
92
|
+
- - ~>
|
93
93
|
- !ruby/object:Gem::Version
|
94
|
-
version:
|
94
|
+
version: 1.2.0
|
95
95
|
prerelease: false
|
96
96
|
type: :runtime
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
98
|
+
name: faraday_middleware
|
99
99
|
version_requirements: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- -
|
101
|
+
- - ~>
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version:
|
103
|
+
version: 0.9.0
|
104
104
|
requirement: !ruby/object:Gem::Requirement
|
105
105
|
requirements:
|
106
|
-
- -
|
106
|
+
- - ~>
|
107
107
|
- !ruby/object:Gem::Version
|
108
|
-
version:
|
108
|
+
version: 0.9.0
|
109
109
|
prerelease: false
|
110
110
|
type: :runtime
|
111
111
|
- !ruby/object:Gem::Dependency
|
@@ -274,18 +274,11 @@ files:
|
|
274
274
|
- lib/mode/api/resource.rb
|
275
275
|
- lib/mode/auth/access_token.rb
|
276
276
|
- lib/mode/cli.rb
|
277
|
-
- lib/mode/cli/analyze.rb
|
278
277
|
- lib/mode/cli/base.rb
|
279
278
|
- lib/mode/cli/connect.rb
|
280
|
-
- lib/mode/cli/helpers.rb
|
281
|
-
- lib/mode/cli/import.rb
|
282
279
|
- lib/mode/cli/login.rb
|
283
|
-
- lib/mode/cli/package.rb
|
284
|
-
- lib/mode/commands/analyze_field.rb
|
285
|
-
- lib/mode/commands/analyze_schema.rb
|
286
280
|
- lib/mode/commands/connect.rb
|
287
281
|
- lib/mode/commands/helpers.rb
|
288
|
-
- lib/mode/commands/import.rb
|
289
282
|
- lib/mode/commands/login.rb
|
290
283
|
- lib/mode/config.rb
|
291
284
|
- lib/mode/connector/commands/select_report_run_dataset.rb
|
@@ -305,17 +298,15 @@ files:
|
|
305
298
|
- lib/mode/logger.rb
|
306
299
|
- lib/mode/version.rb
|
307
300
|
- mode.gemspec
|
301
|
+
- releases/mode-ruby-0.0.18.jar
|
308
302
|
- script/console.rb
|
309
303
|
- spec/api/form_spec.rb
|
310
304
|
- spec/api/link_spec.rb
|
311
305
|
- spec/api/request_spec.rb
|
312
306
|
- spec/api/resource_spec.rb
|
313
307
|
- spec/auth/access_token_spec.rb
|
314
|
-
- spec/commands/analyze_field_spec.rb
|
315
|
-
- spec/commands/analyze_schema_spec.rb
|
316
308
|
- spec/commands/connect_spec.rb
|
317
309
|
- spec/commands/helpers_spec.rb
|
318
|
-
- spec/commands/import_spec.rb
|
319
310
|
- spec/commands/login_spec.rb
|
320
311
|
- spec/config_spec.rb
|
321
312
|
- spec/connector/commands/select_report_run_dataset_spec.rb
|
@@ -372,11 +363,8 @@ test_files:
|
|
372
363
|
- spec/api/request_spec.rb
|
373
364
|
- spec/api/resource_spec.rb
|
374
365
|
- spec/auth/access_token_spec.rb
|
375
|
-
- spec/commands/analyze_field_spec.rb
|
376
|
-
- spec/commands/analyze_schema_spec.rb
|
377
366
|
- spec/commands/connect_spec.rb
|
378
367
|
- spec/commands/helpers_spec.rb
|
379
|
-
- spec/commands/import_spec.rb
|
380
368
|
- spec/commands/login_spec.rb
|
381
369
|
- spec/config_spec.rb
|
382
370
|
- spec/connector/commands/select_report_run_dataset_spec.rb
|
data/lib/mode/cli/analyze.rb
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
module Mode
|
2
|
-
module CLI
|
3
|
-
class Base < Thor
|
4
|
-
desc "analyze PATH [--field=POSITION] [--match-type=TYPE]", "Analyze a CSV file"
|
5
|
-
option :sample, :banner => 'RATE', :desc => "Proportion of rows to inspect. Example: 0.5"
|
6
|
-
option :field, :banner => 'POSITION', :desc => "Field positions begin at 0. Example: 10"
|
7
|
-
option :match_type, :banner => 'TYPE', :desc => "One of the following: string, number, integer, datetime, boolean"
|
8
|
-
# option :keys, :banner => 'POSITIONS (ex: 0,2)', :default => String.new
|
9
|
-
def analyze(path = nil)
|
10
|
-
#keys = options[:keys].split(',').collect(&:strip).collect(&:to_i)
|
11
|
-
|
12
|
-
if field_pos = options[:field]
|
13
|
-
Mode::Commands::AnalyzeField.new(path, field_pos, options).execute
|
14
|
-
else
|
15
|
-
Mode::Commands::AnalyzeSchema.new(path, options).execute
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
data/lib/mode/cli/helpers.rb
DELETED
data/lib/mode/cli/import.rb
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
module Mode
|
2
|
-
module CLI
|
3
|
-
class Base < Thor
|
4
|
-
desc "import SOURCE ACCOUNT/TABLENAME [--replace]", "Import a flat file into the Mode data warehouse", :hide => true
|
5
|
-
long_desc <<-LONGDESC
|
6
|
-
The import commands allows you to create or replace tables
|
7
|
-
in the Mode data warehouse with data from flat files and
|
8
|
-
data packages. The default action is create with an optional
|
9
|
-
flag to replace the table.
|
10
|
-
|
11
|
-
Data can be imported from CSV files or data packages with the following command
|
12
|
-
|
13
|
-
1. CSV File
|
14
|
-
\x5> $ mode import gdp_quarterly.csv besquared/gdp_quarterly
|
15
|
-
|
16
|
-
2. Data Package File
|
17
|
-
\x5> $ mode import gdp/data/quarterly.json besqaured/quarterly_gdp
|
18
|
-
|
19
|
-
LONGDESC
|
20
|
-
option :replace, :type => :boolean
|
21
|
-
option :primary_key, :banner => 'pos1[,pos2,...] (ex: 0,2)'
|
22
|
-
def import(source, table)
|
23
|
-
unless valid_table?(table)
|
24
|
-
error "Error: Invalid account or table name given"
|
25
|
-
return
|
26
|
-
end
|
27
|
-
|
28
|
-
account, table_name = *table.split('/')
|
29
|
-
Mode::Commands::Import.new(source, account, table_name).execute
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
data/lib/mode/cli/package.rb
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
module Mode
|
2
|
-
module CLI
|
3
|
-
class Base < Thor
|
4
|
-
desc "package SOURCE PACKAGEPATH [--name=NAME] [--resource-name=NAME]", "Creates a new data package from a csv file"
|
5
|
-
option :name, :desc => 'The name of the package'
|
6
|
-
option :resource_name, :default => 'data', :desc => 'The name of the resource in package'
|
7
|
-
def package(source, package_path)
|
8
|
-
name = options[:name] || package_path.split(File::Separator).last
|
9
|
-
Mode::Commands::Package.new(source, package_path, name, options[:resource_name]).execute
|
10
|
-
end
|
11
|
-
end
|
12
|
-
end
|
13
|
-
end
|
@@ -1,60 +0,0 @@
|
|
1
|
-
require 'terminal-table'
|
2
|
-
|
3
|
-
module Mode
|
4
|
-
module Commands
|
5
|
-
class AnalyzeField
|
6
|
-
include Mode::Commands::Helpers
|
7
|
-
|
8
|
-
attr_accessor :path
|
9
|
-
attr_accessor :field_pos
|
10
|
-
attr_accessor :options
|
11
|
-
|
12
|
-
def initialize(path, field_pos, options = {})
|
13
|
-
@path = path
|
14
|
-
@field_pos = field_pos.to_i
|
15
|
-
@options = options
|
16
|
-
end
|
17
|
-
|
18
|
-
def execute
|
19
|
-
if path.nil? || !File.exist?(path)
|
20
|
-
puts "Error: Couldn't find file at #{path}"
|
21
|
-
return
|
22
|
-
end
|
23
|
-
|
24
|
-
csv = DataKit::CSV::Parser.new(path)
|
25
|
-
|
26
|
-
puts "Analyzing field #{field_pos} at #{path || 'input'}"
|
27
|
-
|
28
|
-
analysis, total_time = timer_block do
|
29
|
-
DataKit::CSV::FieldAnalyzer.analyze(csv, field_pos, {
|
30
|
-
:match_type => match_type, :sampling_rate => 1
|
31
|
-
})
|
32
|
-
end
|
33
|
-
|
34
|
-
puts "Analyzed #{analysis.row_count} rows in #{'%.2f' % total_time} seconds\n"
|
35
|
-
|
36
|
-
display(analysis)
|
37
|
-
end
|
38
|
-
|
39
|
-
private
|
40
|
-
|
41
|
-
def display(analysis)
|
42
|
-
table = Terminal::Table.new(:headings => [
|
43
|
-
'Row No.', 'Type', 'Value'
|
44
|
-
])
|
45
|
-
|
46
|
-
analysis.types.each do |type, rows|
|
47
|
-
rows.each do |row_num|
|
48
|
-
table.add_row [row_num, type, analysis.value_at(row_num)]
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
puts table
|
53
|
-
end
|
54
|
-
|
55
|
-
def match_type
|
56
|
-
options[:match_type] ? options[:match_type].to_sym : :any
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
@@ -1,111 +0,0 @@
|
|
1
|
-
require 'terminal-table'
|
2
|
-
|
3
|
-
module Mode
|
4
|
-
module Commands
|
5
|
-
class AnalyzeSchema
|
6
|
-
include Mode::Commands::Helpers
|
7
|
-
|
8
|
-
attr_accessor :path
|
9
|
-
attr_accessor :options
|
10
|
-
|
11
|
-
def initialize(path, options = {})
|
12
|
-
@path = path
|
13
|
-
@options = options
|
14
|
-
end
|
15
|
-
|
16
|
-
def execute
|
17
|
-
if path.nil? || !File.exist?(path)
|
18
|
-
puts "Error: Couldn't find file at #{path}"
|
19
|
-
return
|
20
|
-
end
|
21
|
-
|
22
|
-
csv = build_csv
|
23
|
-
analysis = build_analysis(csv)
|
24
|
-
display_analysis(analysis)
|
25
|
-
end
|
26
|
-
|
27
|
-
private
|
28
|
-
|
29
|
-
def build_csv
|
30
|
-
DataKit::CSV::Parser.new(path)
|
31
|
-
end
|
32
|
-
|
33
|
-
def build_analysis(csv)
|
34
|
-
puts "Analyzing #{path || 'input'} (Sampling #{'%.2f' % (100 * sampling_rate)}%)..."
|
35
|
-
|
36
|
-
analysis, total_time = timer_block do
|
37
|
-
DataKit::CSV::SchemaAnalyzer.analyze(csv, :sampling_rate => sampling_rate)
|
38
|
-
end
|
39
|
-
|
40
|
-
puts "Analyzed #{analysis.sample_count} of #{analysis.row_count} rows in #{'%.2f' % total_time} seconds\n"
|
41
|
-
|
42
|
-
analysis
|
43
|
-
end
|
44
|
-
|
45
|
-
def sampling_rate
|
46
|
-
file_size = File.size(path)
|
47
|
-
(options[:sample] || DataKit::CSV::SchemaAnalyzer.sampling_rate(file_size)).to_f
|
48
|
-
end
|
49
|
-
|
50
|
-
def display_analysis(analysis)
|
51
|
-
table = build_table
|
52
|
-
populate_table(analysis, table)
|
53
|
-
puts table
|
54
|
-
end
|
55
|
-
|
56
|
-
def build_table
|
57
|
-
Terminal::Table.new(:headings => [
|
58
|
-
'Field No.', 'Field', 'Type',
|
59
|
-
'String (%)', 'Integer (%)', 'Number (%)',
|
60
|
-
'Date/Time (%)', 'Boolean (%)', 'Empty (%)'
|
61
|
-
])
|
62
|
-
end
|
63
|
-
|
64
|
-
def populate_table(analysis, table)
|
65
|
-
analysis.fields.each_with_index do |field_name, index|
|
66
|
-
build_table_row(analysis, table, field_name, index)
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
def build_table_row(analysis, table, field_name, index)
|
71
|
-
row = [index, field_name]
|
72
|
-
|
73
|
-
append_row_type(analysis, field_name, row)
|
74
|
-
append_row_type_counts(analysis, field_name, row)
|
75
|
-
|
76
|
-
table.add_row(row)
|
77
|
-
end
|
78
|
-
|
79
|
-
def append_row_type(analysis, field_name, row)
|
80
|
-
field_type = analysis.type?(field_name)
|
81
|
-
|
82
|
-
if analysis.has_single_type?(field_name)
|
83
|
-
row << field_type
|
84
|
-
elsif analysis.has_only_numeric_types?(field_name)
|
85
|
-
row << field_type
|
86
|
-
else
|
87
|
-
row << '** ' + field_type.to_s
|
88
|
-
end
|
89
|
-
end
|
90
|
-
|
91
|
-
def append_row_type_counts(analysis, field_name, row)
|
92
|
-
DataKit::Dataset::Field::Types.each do |type|
|
93
|
-
type_count = analysis.type_count(field_name, type)
|
94
|
-
row << format_percentage_cell(type_count, analysis.sample_count)
|
95
|
-
end
|
96
|
-
end
|
97
|
-
|
98
|
-
def format_percentage_cell(numerator, denominator)
|
99
|
-
cell = { :alignment => :right }
|
100
|
-
|
101
|
-
if numerator == 0
|
102
|
-
cell[:value] = nil
|
103
|
-
else
|
104
|
-
cell[:value] = '%.2f' % (100 * (numerator / denominator.to_f)) + '%'
|
105
|
-
end
|
106
|
-
|
107
|
-
cell
|
108
|
-
end
|
109
|
-
end
|
110
|
-
end
|
111
|
-
end
|
data/lib/mode/commands/import.rb
DELETED
@@ -1,224 +0,0 @@
|
|
1
|
-
module Mode
|
2
|
-
module Commands
|
3
|
-
class Import
|
4
|
-
include Mode::Commands::Helpers
|
5
|
-
|
6
|
-
attr_reader :source
|
7
|
-
attr_reader :account
|
8
|
-
attr_reader :table_name
|
9
|
-
|
10
|
-
def initialize(source, account, table_name)
|
11
|
-
@source = source
|
12
|
-
@account = account
|
13
|
-
@table_name = table_name
|
14
|
-
end
|
15
|
-
|
16
|
-
def execute
|
17
|
-
validate_config!
|
18
|
-
|
19
|
-
package = make_package
|
20
|
-
uploaded = upload_package(package)
|
21
|
-
imported = import_package(uploaded)
|
22
|
-
end
|
23
|
-
|
24
|
-
def make_package
|
25
|
-
find_or_build_package(source, :name => table_name)
|
26
|
-
end
|
27
|
-
|
28
|
-
def upload_package(package)
|
29
|
-
resource = package.resources.first
|
30
|
-
file_path = File.join(resource.base_path, resource.path)
|
31
|
-
|
32
|
-
replace_resource_path!(resource)
|
33
|
-
|
34
|
-
Mode::API::Request.post(upload_path, :package => {
|
35
|
-
:contents => {
|
36
|
-
'datapackage.json' => package.to_json,
|
37
|
-
resource.path => Faraday::UploadIO.new(file_path, 'application/octet-stream')
|
38
|
-
}, :name => table_name
|
39
|
-
})
|
40
|
-
end
|
41
|
-
|
42
|
-
def import_package(uploaded)
|
43
|
-
resource = parse_uploaded_response(uploaded)
|
44
|
-
submit_import_form(resource, { 'table_name' => table_name })
|
45
|
-
end
|
46
|
-
|
47
|
-
def target_resource_path(path)
|
48
|
-
self.class.target_resource_path(path)
|
49
|
-
end
|
50
|
-
|
51
|
-
private
|
52
|
-
|
53
|
-
def upload_path
|
54
|
-
Mode::API::Request.packages_path
|
55
|
-
end
|
56
|
-
|
57
|
-
def find_or_build_package(source, options = {})
|
58
|
-
self.class.find_or_build_package(source, options)
|
59
|
-
end
|
60
|
-
|
61
|
-
# We replace the resource path with the desired path on the remote end
|
62
|
-
def replace_resource_path!(resource)
|
63
|
-
resource.path = target_resource_path(resource.path)
|
64
|
-
end
|
65
|
-
|
66
|
-
def submit_import_form(resource, params)
|
67
|
-
resource.forms('import').submit!(:import => params)
|
68
|
-
end
|
69
|
-
|
70
|
-
def parse_uploaded_response(content)
|
71
|
-
resource = Mode::API::Resource.new(content)
|
72
|
-
end
|
73
|
-
|
74
|
-
class << self
|
75
|
-
#
|
76
|
-
# Utilities that can be useful elsewhere
|
77
|
-
#
|
78
|
-
|
79
|
-
def find_package(source, path = nil)
|
80
|
-
path = path || search_path(source)
|
81
|
-
|
82
|
-
if path =~ /\/data$/
|
83
|
-
parent = parent_path(path)
|
84
|
-
find_package(source, parent)
|
85
|
-
elsif package_exists?(path)
|
86
|
-
DataPackage::Package.open(path)
|
87
|
-
else
|
88
|
-
nil
|
89
|
-
end
|
90
|
-
end
|
91
|
-
|
92
|
-
def build_package(source, options = {})
|
93
|
-
name = options[:name] || 'data'
|
94
|
-
dest = options[:dest] || tmpdir
|
95
|
-
|
96
|
-
if File.exist?(source)
|
97
|
-
converter = convert(source)
|
98
|
-
DataPackage::Package.new(dest).tap do |package|
|
99
|
-
package.name = name
|
100
|
-
package.version = '0.0.1'
|
101
|
-
package.resources << build_resource(converter)
|
102
|
-
end
|
103
|
-
else
|
104
|
-
nil
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
def find_or_build_package(source, options = {})
|
109
|
-
package = find_package(source)
|
110
|
-
|
111
|
-
if package.nil?
|
112
|
-
build_opts = {
|
113
|
-
:name => options[:name],
|
114
|
-
:dest => options[:dest]
|
115
|
-
}
|
116
|
-
package = build_package(source, build_opts)
|
117
|
-
else
|
118
|
-
package = prune_package(source, package)
|
119
|
-
end
|
120
|
-
|
121
|
-
package
|
122
|
-
end
|
123
|
-
|
124
|
-
def target_resource_path(path = nil)
|
125
|
-
"data/index#{path.nil? ? '.csv' : File.extname(path)}"
|
126
|
-
end
|
127
|
-
|
128
|
-
private
|
129
|
-
|
130
|
-
#
|
131
|
-
# Package Building
|
132
|
-
#
|
133
|
-
|
134
|
-
def build_resource(converter)
|
135
|
-
DataPackage::Resource.new(tmpdir, {
|
136
|
-
'name' => 'index.csv',
|
137
|
-
'path' => target_resource_path,
|
138
|
-
'size' => File.size?(target),
|
139
|
-
'schema' => build_schema(converter)
|
140
|
-
})
|
141
|
-
end
|
142
|
-
|
143
|
-
def build_schema(converter)
|
144
|
-
{
|
145
|
-
'fields' => build_fields(converter),
|
146
|
-
'dialect' => DataPackage::Dialect.new
|
147
|
-
}
|
148
|
-
end
|
149
|
-
|
150
|
-
def build_fields(converter)
|
151
|
-
converter.field_types.inject([]) do |fields, (field_name, field_type)|
|
152
|
-
fields << {'name' => field_name, 'type' => field_type.to_s}
|
153
|
-
end
|
154
|
-
end
|
155
|
-
|
156
|
-
#
|
157
|
-
# Data Conversion
|
158
|
-
#
|
159
|
-
|
160
|
-
def convert(source)
|
161
|
-
csv = DataKit::CSV::Parser.new(source)
|
162
|
-
analysis = DataKit::CSV::SchemaAnalyzer.analyze(csv, :sampling_rate => 1.0)
|
163
|
-
DataKit::CSV::Converter.convert(csv, analysis, target)
|
164
|
-
end
|
165
|
-
|
166
|
-
def tmpdir
|
167
|
-
@tmpdir ||= Dir.mktmpdir
|
168
|
-
end
|
169
|
-
|
170
|
-
def tmppath(path)
|
171
|
-
File.join(tmpdir, path)
|
172
|
-
end
|
173
|
-
|
174
|
-
def target(path = 'data', name = 'index.csv')
|
175
|
-
full_path = tmppath(path)
|
176
|
-
FileUtils.mkdir_p(full_path)
|
177
|
-
@target ||= File.join(full_path, name)
|
178
|
-
end
|
179
|
-
|
180
|
-
#
|
181
|
-
# Checks and Validations
|
182
|
-
#
|
183
|
-
|
184
|
-
def search_path(source)
|
185
|
-
File.file?(source) ? File.dirname(source) : source
|
186
|
-
end
|
187
|
-
|
188
|
-
def package_exists?(path)
|
189
|
-
DataPackage::Package.exist?(path)
|
190
|
-
end
|
191
|
-
|
192
|
-
def parent_path(path)
|
193
|
-
File.expand_path(File.join(path, '..'))
|
194
|
-
end
|
195
|
-
|
196
|
-
def resource_path(path)
|
197
|
-
matched = path.match(/((data\/)?[^\/]+)$/)
|
198
|
-
matched.nil? ? matched : matched.to_a.first
|
199
|
-
end
|
200
|
-
|
201
|
-
def first_resource(package)
|
202
|
-
package.resources.find{ |r| not r.path.nil? }
|
203
|
-
end
|
204
|
-
|
205
|
-
def find_resource(package, path)
|
206
|
-
package.resources.find{ |r| r.path == path }
|
207
|
-
end
|
208
|
-
|
209
|
-
def prune_package(source, package)
|
210
|
-
if File.directory?(source)
|
211
|
-
resource = first_resource(package)
|
212
|
-
elsif File.file?(source)
|
213
|
-
resource_path = resource_path(source)
|
214
|
-
resource = find_resource(package, resource_path)
|
215
|
-
end
|
216
|
-
|
217
|
-
package.resources = []
|
218
|
-
package.resources << resource
|
219
|
-
package
|
220
|
-
end
|
221
|
-
end
|
222
|
-
end
|
223
|
-
end
|
224
|
-
end
|
@@ -1,26 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Mode::Commands::AnalyzeField do
|
4
|
-
let(:tmpdir) { Dir.mktmpdir }
|
5
|
-
|
6
|
-
before do
|
7
|
-
initialize_logger
|
8
|
-
end
|
9
|
-
|
10
|
-
it "should analyze a csv" do
|
11
|
-
path = fixture_path('espn_draft.csv')
|
12
|
-
command = Mode::Commands::AnalyzeField.new(path, 1)
|
13
|
-
|
14
|
-
command.stub(:puts).and_return(true)
|
15
|
-
|
16
|
-
command.execute
|
17
|
-
end
|
18
|
-
|
19
|
-
it "should error out if the csv file doesn't exist" do
|
20
|
-
command = Mode::Commands::AnalyzeField.new("fake.yml", 1)
|
21
|
-
|
22
|
-
command.should_receive(:puts)
|
23
|
-
|
24
|
-
command.execute
|
25
|
-
end
|
26
|
-
end
|
@@ -1,26 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Mode::Commands::AnalyzeSchema do
|
4
|
-
let(:tmpdir) { Dir.mktmpdir }
|
5
|
-
|
6
|
-
before do
|
7
|
-
initialize_logger
|
8
|
-
end
|
9
|
-
|
10
|
-
it "should analyze a csv" do
|
11
|
-
path = fixture_path('espn_draft.csv')
|
12
|
-
command = Mode::Commands::AnalyzeSchema.new(path)
|
13
|
-
|
14
|
-
command.stub(:puts).and_return(true)
|
15
|
-
|
16
|
-
command.execute
|
17
|
-
end
|
18
|
-
|
19
|
-
it "should error out if the csv file doesn't exist" do
|
20
|
-
command = Mode::Commands::AnalyzeSchema.new("fake.yml", keys: [0,1])
|
21
|
-
|
22
|
-
command.should_receive(:puts)
|
23
|
-
|
24
|
-
command.execute
|
25
|
-
end
|
26
|
-
end
|
@@ -1,160 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Mode::Commands::Import do
|
4
|
-
let(:tmpdir) { Dir.mktmpdir }
|
5
|
-
let(:config) { Mode::Config.init(tmpdir) }
|
6
|
-
let(:packages_path) { Mode::API::Request.packages_path }
|
7
|
-
|
8
|
-
before do
|
9
|
-
initialize_logger
|
10
|
-
end
|
11
|
-
|
12
|
-
it "generates target resource path with a csv extension" do
|
13
|
-
source = fixture_path("espn_draft/data.csv")
|
14
|
-
path = Mode::Commands::Import.send(:target_resource_path, source)
|
15
|
-
|
16
|
-
path.should == "data/index.csv"
|
17
|
-
end
|
18
|
-
|
19
|
-
it "generates target resource path with a non-csv extension" do
|
20
|
-
source = fixture_path("espn_draft.json")
|
21
|
-
path = Mode::Commands::Import.send(:target_resource_path, source)
|
22
|
-
|
23
|
-
path.should == "data/index.json"
|
24
|
-
end
|
25
|
-
|
26
|
-
describe 'finding a package from a source file' do
|
27
|
-
it 'returns the package if the file is adjacent to datapackage.json' do
|
28
|
-
source = fixture_path("espn_draft/data.csv")
|
29
|
-
package = Mode::Commands::Import.find_package(source)
|
30
|
-
|
31
|
-
package.base_path.should == fixture_path("espn_draft")
|
32
|
-
end
|
33
|
-
|
34
|
-
it 'returns the package if the file is inside of the data subdirectory' do
|
35
|
-
source = fixture_path("country-codes/data/country-codes.csv")
|
36
|
-
package = Mode::Commands::Import.find_package(source)
|
37
|
-
|
38
|
-
package.base_path.should == fixture_path("country-codes")
|
39
|
-
end
|
40
|
-
|
41
|
-
it 'returns nil if a package is not found' do
|
42
|
-
source = fixture_path("espn_draft.csv")
|
43
|
-
Mode::Commands::Import.find_package(source).should == nil
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
describe 'finding a package from a source directory' do
|
48
|
-
it 'returns the package if it exists in the directory' do
|
49
|
-
source = fixture_path("espn_draft")
|
50
|
-
package = Mode::Commands::Import.find_package(source)
|
51
|
-
|
52
|
-
package.base_path.should == fixture_path("espn_draft")
|
53
|
-
end
|
54
|
-
|
55
|
-
it 'returns nil if the package is not found in the directory' do
|
56
|
-
source = fixture_path
|
57
|
-
|
58
|
-
Mode::Commands::Import.find_package(source).should == nil
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
describe 'building a package from a source file' do
|
63
|
-
it 'build the package' do
|
64
|
-
source = fixture_path("espn_draft.csv")
|
65
|
-
package = Mode::Commands::Import.build_package(source, :dest => tmpdir)
|
66
|
-
|
67
|
-
package.base_path.should == tmpdir
|
68
|
-
end
|
69
|
-
|
70
|
-
it 'returns nil if the file does not exist' do
|
71
|
-
source = fixture_path('nope')
|
72
|
-
Mode::Commands::Import.build_package(source).should == nil
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
describe 'finding or building a package from a source file' do
|
77
|
-
it 'returns the package and resource name for a file in a datapackage' do
|
78
|
-
source = fixture_path("espn_draft/data.csv")
|
79
|
-
package = Mode::Commands::Import.find_or_build_package(source)
|
80
|
-
|
81
|
-
package.resources.first.path.should == 'data.csv'
|
82
|
-
package.base_path.should == fixture_path("espn_draft")
|
83
|
-
end
|
84
|
-
|
85
|
-
it 'returns the package and resource name for a file outisde a datapackage' do
|
86
|
-
source = fixture_path("espn_draft.csv")
|
87
|
-
package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
|
88
|
-
|
89
|
-
package.base_path.should == tmpdir
|
90
|
-
package.resources.first.path.should == 'data/index.csv'
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
describe 'finding or building a package from a source directory' do
|
95
|
-
it 'returns the package and resource name' do
|
96
|
-
source = fixture_path("espn_draft")
|
97
|
-
package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
|
98
|
-
|
99
|
-
package.resources.first.path.should == 'data.csv'
|
100
|
-
package.base_path.should == fixture_path("espn_draft")
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
describe 'uploading a package' do
|
105
|
-
it 'uploads a package' do
|
106
|
-
Mode::API::Request.should_receive(:post).and_return(true)
|
107
|
-
|
108
|
-
source = fixture_path("espn_draft.csv")
|
109
|
-
package = Mode::Commands::Import.find_or_build_package(source, :dest => tmpdir)
|
110
|
-
|
111
|
-
import = Mode::Commands::Import.new(source, 'besquared', 'espn_draft')
|
112
|
-
|
113
|
-
import.upload_package(package).should == true
|
114
|
-
end
|
115
|
-
end
|
116
|
-
|
117
|
-
describe 'executing an import' do
|
118
|
-
it 'finds and upload a package' do
|
119
|
-
Mode::API::Request.should_receive(:post).with(packages_path, :package => {
|
120
|
-
:contents => {
|
121
|
-
'datapackage.json' => an_instance_of(String),
|
122
|
-
'data/index.csv' => an_instance_of(Faraday::UploadIO)
|
123
|
-
}, :name => 'espn_draft'
|
124
|
-
}).and_return(:package)
|
125
|
-
|
126
|
-
source = fixture_path("espn_draft/data.csv")
|
127
|
-
import = Mode::Commands::Import.new(source, 'besquared', 'espn_draft')
|
128
|
-
|
129
|
-
resource = double(:resource)
|
130
|
-
import_params = { 'table_name' => 'espn_draft' }
|
131
|
-
import.should_receive(:parse_uploaded_response).with(:package).and_return(resource)
|
132
|
-
import.should_receive(:submit_import_form).with(resource, import_params).and_return(true)
|
133
|
-
|
134
|
-
import.should_receive(:validate_config!).and_return(true)
|
135
|
-
|
136
|
-
import.execute
|
137
|
-
end
|
138
|
-
|
139
|
-
it 'finds and uploads country codes package' do
|
140
|
-
Mode::API::Request.should_receive(:post).with(packages_path, :package => {
|
141
|
-
:contents => {
|
142
|
-
'datapackage.json' => an_instance_of(String),
|
143
|
-
'data/index.csv' => an_instance_of(Faraday::UploadIO)
|
144
|
-
}, :name => 'country_codes'
|
145
|
-
}).and_return(:package)
|
146
|
-
|
147
|
-
source = fixture_path("country-codes/data/country-codes.csv")
|
148
|
-
import = Mode::Commands::Import.new(source, 'besquared', 'country_codes')
|
149
|
-
|
150
|
-
resource = double(:resource)
|
151
|
-
import_params = { 'table_name' => 'country_codes' }
|
152
|
-
import.should_receive(:parse_uploaded_response).with(:package).and_return(resource)
|
153
|
-
import.should_receive(:submit_import_form).with(resource, import_params).and_return(true)
|
154
|
-
|
155
|
-
import.should_receive(:validate_config!).and_return(true)
|
156
|
-
|
157
|
-
import.execute
|
158
|
-
end
|
159
|
-
end
|
160
|
-
end
|