nsrr 0.4.0.beta1 → 6.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +19 -1
- data/LICENSE +20 -111
- data/README.md +14 -52
- data/Rakefile +4 -4
- data/bin/nsrr +1 -1
- data/lib/nsrr.rb +11 -11
- data/lib/nsrr/commands/console.rb +3 -3
- data/lib/nsrr/commands/download.rb +39 -20
- data/lib/nsrr/commands/update.rb +8 -8
- data/lib/nsrr/helpers/authorization.rb +9 -8
- data/lib/nsrr/helpers/color.rb +100 -0
- data/lib/nsrr/helpers/constants.rb +1 -1
- data/lib/nsrr/helpers/download_request.rb +9 -12
- data/lib/nsrr/helpers/hash_helper.rb +4 -1
- data/lib/nsrr/helpers/json_request.rb +32 -19
- data/lib/nsrr/models/all.rb +1 -1
- data/lib/nsrr/models/dataset.rb +36 -37
- data/lib/nsrr/models/file.rb +34 -33
- data/lib/nsrr/version.rb +4 -4
- data/nsrr.gemspec +18 -15
- metadata +15 -29
data/lib/nsrr/commands/update.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require "nsrr/helpers/color"
|
4
|
+
require "nsrr/helpers/json_request"
|
5
5
|
|
6
6
|
module Nsrr
|
7
7
|
module Commands
|
@@ -17,19 +17,19 @@ module Nsrr
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def start
|
20
|
-
(json, _status) = Nsrr::Helpers::JsonRequest.get(
|
20
|
+
(json, _status) = Nsrr::Helpers::JsonRequest.get("https://rubygems.org/api/v1/gems/nsrr.json")
|
21
21
|
if json
|
22
|
-
if json[
|
23
|
-
puts
|
22
|
+
if json["version"] == Nsrr::VERSION::STRING
|
23
|
+
puts "The nsrr gem is " + "up-to-date".green + "!"
|
24
24
|
else
|
25
25
|
puts
|
26
|
-
puts "A newer version (v#{json[
|
26
|
+
puts "A newer version (v#{json["version"]}) is available! Type the following command to update:"
|
27
27
|
puts
|
28
|
-
puts
|
28
|
+
puts " gem install nsrr --no-document".white
|
29
29
|
puts
|
30
30
|
end
|
31
31
|
else
|
32
|
-
puts
|
32
|
+
puts "Unable to connect to RubyGems.org. Please try again later."
|
33
33
|
end
|
34
34
|
end
|
35
35
|
end
|
@@ -1,22 +1,23 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "nsrr/helpers/color"
|
4
|
+
require "nsrr/helpers/json_request"
|
4
5
|
|
5
6
|
module Nsrr
|
6
7
|
module Helpers
|
7
8
|
# Helper to verify that user is authenticated.
|
8
9
|
class Authorization
|
9
10
|
def self.get_token(token)
|
10
|
-
puts
|
11
|
-
puts
|
12
|
-
print
|
13
|
-
token = STDIN.noecho(&:gets).chomp if token.to_s.strip ==
|
11
|
+
puts " Get your token here: " + "#{Nsrr::WEBSITE}/token".bg_gray.blue.underline
|
12
|
+
puts " Your input is hidden while entering token.".white
|
13
|
+
print " Enter your token: "
|
14
|
+
token = STDIN.noecho(&:gets).chomp if token.to_s.strip == ""
|
14
15
|
token.strip!
|
15
16
|
(response, _status) = Nsrr::Helpers::JsonRequest.get("#{Nsrr::WEBSITE}/api/v1/account/profile.json", auth_token: token)
|
16
|
-
if response.is_a?(Hash) && response[
|
17
|
-
puts
|
17
|
+
if response.is_a?(Hash) && response["authenticated"]
|
18
|
+
puts "AUTHORIZED".green + " as " + "#{response["first_name"]} #{response["last_name"]}".white
|
18
19
|
else
|
19
|
-
puts
|
20
|
+
puts "UNAUTHORIZED".red + " Public Access Only"
|
20
21
|
end
|
21
22
|
token
|
22
23
|
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Adds ANSI colors to string class.
|
4
|
+
class String
|
5
|
+
def black
|
6
|
+
"\e[30m#{self}\e[0m"
|
7
|
+
end
|
8
|
+
|
9
|
+
def red
|
10
|
+
"\e[31m#{self}\e[0m"
|
11
|
+
end
|
12
|
+
|
13
|
+
def green
|
14
|
+
"\e[32m#{self}\e[0m"
|
15
|
+
end
|
16
|
+
|
17
|
+
def brown
|
18
|
+
"\e[33m#{self}\e[0m"
|
19
|
+
end
|
20
|
+
|
21
|
+
def yellow
|
22
|
+
brown
|
23
|
+
end
|
24
|
+
|
25
|
+
def blue
|
26
|
+
"\e[34m#{self}\e[0m"
|
27
|
+
end
|
28
|
+
|
29
|
+
def magenta
|
30
|
+
"\e[35m#{self}\e[0m"
|
31
|
+
end
|
32
|
+
|
33
|
+
def cyan
|
34
|
+
"\e[36m#{self}\e[0m"
|
35
|
+
end
|
36
|
+
|
37
|
+
def gray
|
38
|
+
"\e[37m#{self}\e[0m"
|
39
|
+
end
|
40
|
+
|
41
|
+
def white
|
42
|
+
"\e[39m#{bold}\e[0m"
|
43
|
+
end
|
44
|
+
|
45
|
+
def bg_black
|
46
|
+
"\e[40m#{self}\e[0m"
|
47
|
+
end
|
48
|
+
|
49
|
+
def bg_red
|
50
|
+
"\e[41m#{self}\e[0m"
|
51
|
+
end
|
52
|
+
|
53
|
+
def bg_green
|
54
|
+
"\e[42m#{self}\e[0m"
|
55
|
+
end
|
56
|
+
|
57
|
+
def bg_brown
|
58
|
+
"\e[43m#{self}\e[0m"
|
59
|
+
end
|
60
|
+
|
61
|
+
def bg_blue
|
62
|
+
"\e[44m#{self}\e[0m"
|
63
|
+
end
|
64
|
+
|
65
|
+
def bg_magenta
|
66
|
+
"\e[45m#{self}\e[0m"
|
67
|
+
end
|
68
|
+
|
69
|
+
def bg_cyan
|
70
|
+
"\e[46m#{self}\e[0m"
|
71
|
+
end
|
72
|
+
|
73
|
+
def bg_gray
|
74
|
+
"\e[47m#{self}\e[0m"
|
75
|
+
end
|
76
|
+
|
77
|
+
def bold
|
78
|
+
"\e[1m#{self}\e[22m"
|
79
|
+
end
|
80
|
+
|
81
|
+
def italic
|
82
|
+
"\e[3m#{self}\e[23m"
|
83
|
+
end
|
84
|
+
|
85
|
+
def underline
|
86
|
+
"\e[4m#{self}\e[24m"
|
87
|
+
end
|
88
|
+
|
89
|
+
def blink
|
90
|
+
"\e[5m#{self}\e[25m"
|
91
|
+
end
|
92
|
+
|
93
|
+
def reverse_color
|
94
|
+
"\e[7m#{self}\e[27m"
|
95
|
+
end
|
96
|
+
|
97
|
+
def colorless
|
98
|
+
gsub(/\e\[\d{1,2}m/, "")
|
99
|
+
end
|
100
|
+
end
|
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "openssl"
|
4
|
+
require "net/http"
|
5
|
+
require "uri"
|
6
6
|
|
7
7
|
module Nsrr
|
8
8
|
module Helpers
|
@@ -17,34 +17,31 @@ module Nsrr
|
|
17
17
|
attr_reader :url, :error, :file_size
|
18
18
|
|
19
19
|
def initialize(url, download_folder)
|
20
|
-
|
21
|
-
@url = URI.parse(escaped_url)
|
20
|
+
@url = URI.parse(url)
|
22
21
|
@http = Net::HTTP.new(@url.host, @url.port)
|
23
|
-
if @url.scheme ==
|
22
|
+
if @url.scheme == "https"
|
24
23
|
@http.use_ssl = true
|
25
24
|
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
26
25
|
end
|
27
26
|
@download_folder = download_folder
|
28
27
|
@file_size = 0
|
29
|
-
rescue
|
30
|
-
@error = 'Invalid Token'
|
31
28
|
end
|
32
29
|
|
33
30
|
# Writes file segments to disk immediately instead of storing in memory
|
34
31
|
def get
|
35
32
|
return unless @error.nil?
|
36
|
-
local_file = ::File.open(@download_folder,
|
33
|
+
local_file = ::File.open(@download_folder, "wb")
|
37
34
|
partial = true
|
38
35
|
@http.request_get(@url.path) do |response|
|
39
36
|
case response.code
|
40
|
-
when
|
37
|
+
when "200"
|
41
38
|
response.read_body do |segment|
|
42
39
|
local_file.write(segment)
|
43
40
|
end
|
44
41
|
@file_size = ::File.size(@download_folder)
|
45
42
|
partial = false
|
46
|
-
when
|
47
|
-
@error =
|
43
|
+
when "302"
|
44
|
+
@error = "Token Not Authorized to Access Specified File"
|
48
45
|
else
|
49
46
|
@error = "#{response.code} #{response.class.name}"
|
50
47
|
end
|
@@ -5,7 +5,10 @@ module Nsrr
|
|
5
5
|
# Converts hash with string keys into hash with keys as symbols.
|
6
6
|
class HashHelper
|
7
7
|
def self.symbolize_keys(hash)
|
8
|
-
hash.inject({})
|
8
|
+
hash.inject({}) do |memo, (k, v)|
|
9
|
+
memo[k.to_sym] = v
|
10
|
+
memo
|
11
|
+
end
|
9
12
|
end
|
10
13
|
end
|
11
14
|
end
|
@@ -1,9 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
3
|
+
require "openssl"
|
4
|
+
require "net/http"
|
5
|
+
require "json"
|
6
|
+
require "cgi"
|
7
|
+
|
8
|
+
require "nsrr/helpers/color"
|
7
9
|
|
8
10
|
module Nsrr
|
9
11
|
module Helpers
|
@@ -30,41 +32,52 @@ module Nsrr
|
|
30
32
|
@url = URI.parse(url)
|
31
33
|
|
32
34
|
@http = Net::HTTP.new(@url.host, @url.port)
|
33
|
-
if @url.scheme ==
|
35
|
+
if @url.scheme == "https"
|
34
36
|
@http.use_ssl = true
|
35
37
|
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
36
38
|
end
|
37
39
|
rescue
|
38
40
|
@error = "Invalid URL: #{url.inspect}"
|
39
|
-
puts @error.
|
41
|
+
puts @error.red
|
40
42
|
end
|
41
43
|
|
42
44
|
def get
|
43
45
|
return unless @error.nil?
|
46
|
+
|
44
47
|
full_path = @url.path
|
45
|
-
query = ([@url.query] + @params).flatten.compact.join(
|
46
|
-
full_path += "?#{query}" if query.to_s !=
|
48
|
+
query = ([@url.query] + @params).flatten.compact.join("&")
|
49
|
+
full_path += "?#{query}" if query.to_s != ""
|
47
50
|
response = @http.start do |http|
|
48
51
|
http.get(full_path)
|
49
52
|
end
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
+
if response.body
|
54
|
+
[JSON.parse(response.body), response]
|
55
|
+
else
|
56
|
+
[nil, response]
|
57
|
+
end
|
58
|
+
rescue => e # JSON::ParserError, Net::ReadTimeout
|
59
|
+
puts "GET error: #{e}".red
|
60
|
+
[nil, nil]
|
53
61
|
end
|
54
62
|
|
55
63
|
def post
|
56
64
|
return unless @error.nil?
|
65
|
+
|
57
66
|
response = @http.start do |http|
|
58
|
-
http.post(@url.path, @params.flatten.compact.join(
|
67
|
+
http.post(@url.path, @params.flatten.compact.join("&"))
|
68
|
+
end
|
69
|
+
if response.body
|
70
|
+
[JSON.parse(response.body), response]
|
71
|
+
else
|
72
|
+
[nil, response]
|
59
73
|
end
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
nil
|
74
|
+
rescue => e # JSON::ParserError, Net::ReadTimeout
|
75
|
+
puts "POST error: #{e}".red
|
76
|
+
[nil, nil]
|
64
77
|
end
|
65
78
|
|
66
79
|
def patch
|
67
|
-
@params <<
|
80
|
+
@params << "_method=patch"
|
68
81
|
post
|
69
82
|
end
|
70
83
|
|
@@ -79,10 +92,10 @@ module Nsrr
|
|
79
92
|
if value.is_a? Hash
|
80
93
|
value.collect do |k,v|
|
81
94
|
key_value_to_string(k, v, current_scope)
|
82
|
-
end.join(
|
95
|
+
end.join("&")
|
83
96
|
elsif value.is_a? Array
|
84
97
|
value.collect do |v|
|
85
|
-
key_value_to_string(
|
98
|
+
key_value_to_string("", v, current_scope)
|
86
99
|
end
|
87
100
|
else
|
88
101
|
"#{current_scope}=#{CGI.escape(value.to_s)}"
|
data/lib/nsrr/models/all.rb
CHANGED
data/lib/nsrr/models/dataset.rb
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require 'io/console'
|
3
|
+
require "fileutils"
|
4
|
+
require "irb"
|
5
|
+
require "io/console"
|
7
6
|
|
8
|
-
require
|
9
|
-
require
|
10
|
-
require
|
11
|
-
require
|
7
|
+
require "nsrr/helpers/constants"
|
8
|
+
require "nsrr/helpers/color"
|
9
|
+
require "nsrr/helpers/hash_helper"
|
10
|
+
require "nsrr/helpers/json_request"
|
11
|
+
require "nsrr/helpers/authorization"
|
12
12
|
|
13
|
-
require
|
13
|
+
require "nsrr/models/file"
|
14
14
|
|
15
15
|
module Nsrr
|
16
16
|
module Models
|
@@ -18,20 +18,19 @@ module Nsrr
|
|
18
18
|
# allowing dataset files to be downloaded.
|
19
19
|
class Dataset
|
20
20
|
def self.find(slug, token = nil)
|
21
|
-
(json, _status) = Nsrr::Helpers::JsonRequest.get(
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
end
|
21
|
+
(json, _status) = Nsrr::Helpers::JsonRequest.get(
|
22
|
+
"#{Nsrr::WEBSITE}/api/v1/datasets/#{slug}.json",
|
23
|
+
auth_token: token
|
24
|
+
)
|
25
|
+
new(json, token) if json
|
27
26
|
end
|
28
27
|
|
29
28
|
attr_accessor :download_token
|
30
29
|
attr_reader :slug, :name
|
31
30
|
|
32
31
|
def initialize(json = {}, token = nil)
|
33
|
-
@slug = json[
|
34
|
-
@name = json[
|
32
|
+
@slug = json["slug"]
|
33
|
+
@name = json["name"]
|
35
34
|
@files = {}
|
36
35
|
@download_token = token
|
37
36
|
@downloaded_folders = []
|
@@ -52,16 +51,16 @@ module Nsrr
|
|
52
51
|
|
53
52
|
# Options include:
|
54
53
|
# method:
|
55
|
-
#
|
56
|
-
#
|
57
|
-
#
|
54
|
+
# "md5" => [default] Checks if a downloaded file exists with the exact md5 as the online version, if so, skips that file
|
55
|
+
# "fresh" => Downloads every file without checking if it was already downloaded
|
56
|
+
# "fast" => Only checks if a download file exists with the same file size as the online version, if so, skips that file
|
58
57
|
# depth:
|
59
|
-
#
|
60
|
-
#
|
58
|
+
# "recursive" => [default] Downloads files in selected path folder and all subfolders
|
59
|
+
# "shallow" => Only downloads files in selected path folder
|
61
60
|
def download(full_path = nil, *args)
|
62
61
|
options = Nsrr::Helpers::HashHelper.symbolize_keys(args.first || {})
|
63
|
-
options[:method] ||=
|
64
|
-
options[:depth] ||=
|
62
|
+
options[:method] ||= "md5"
|
63
|
+
options[:depth] ||= "recursive"
|
65
64
|
@folders_created = 0
|
66
65
|
@files_downloaded = 0
|
67
66
|
@downloaded_bytes = 0
|
@@ -69,9 +68,9 @@ module Nsrr
|
|
69
68
|
@files_failed = 0
|
70
69
|
|
71
70
|
begin
|
72
|
-
puts
|
73
|
-
puts
|
74
|
-
puts
|
71
|
+
puts " File Check: " + options[:method].to_s.white
|
72
|
+
puts " Depth: " + options[:depth].to_s.white
|
73
|
+
puts ""
|
75
74
|
if @download_token.nil?
|
76
75
|
@download_token = Nsrr::Helpers::Authorization.get_token(@download_token)
|
77
76
|
end
|
@@ -80,17 +79,17 @@ module Nsrr
|
|
80
79
|
|
81
80
|
download_helper(full_path, options)
|
82
81
|
rescue Interrupt, IRB::Abort
|
83
|
-
puts "\n Interrupted".
|
82
|
+
puts "\n Interrupted".red
|
84
83
|
end
|
85
84
|
|
86
85
|
@downloaded_megabytes = @downloaded_bytes / (1024 * 1024)
|
87
86
|
|
88
87
|
puts "\nFinished in #{Time.now - @start_time} seconds." if @start_time
|
89
|
-
puts "\n#{@folders_created} folder#{
|
90
|
-
"#{@files_downloaded} file#{
|
91
|
-
"#{@downloaded_megabytes} MiB#{
|
92
|
-
"#{@files_skipped} file#{
|
93
|
-
"#{@files_failed} file#{
|
88
|
+
puts "\n#{@folders_created} folder#{"s" if @folders_created != 1} created".white + ", " +
|
89
|
+
"#{@files_downloaded} file#{"s" if @files_downloaded != 1} downloaded".green + ", " +
|
90
|
+
"#{@downloaded_megabytes} MiB#{"s" if @downloaded_megabytes != 1} downloaded".green + ", " +
|
91
|
+
"#{@files_skipped} file#{"s" if @files_skipped != 1} skipped".blue + ", " +
|
92
|
+
"#{@files_failed} file#{"s" if @files_failed != 1} failed".send(@files_failed.zero? ? :white : :red) + "\n\n"
|
94
93
|
nil
|
95
94
|
end
|
96
95
|
|
@@ -104,16 +103,16 @@ module Nsrr
|
|
104
103
|
current_folder = ::File.join(slug.to_s, file.folder.to_s)
|
105
104
|
result = file.download(options[:method], current_folder, @download_token)
|
106
105
|
case result
|
107
|
-
when
|
106
|
+
when "fail"
|
108
107
|
@files_failed += 1
|
109
|
-
when
|
108
|
+
when "skip"
|
110
109
|
@files_skipped += 1
|
111
110
|
else
|
112
111
|
@files_downloaded += 1
|
113
112
|
@downloaded_bytes += result
|
114
113
|
end
|
115
114
|
end
|
116
|
-
if options[:depth] ==
|
115
|
+
if options[:depth] == "recursive"
|
117
116
|
files(full_path).reject(&:is_file).each do |file|
|
118
117
|
download_helper(file.full_path, options)
|
119
118
|
end
|
@@ -128,7 +127,7 @@ module Nsrr
|
|
128
127
|
end
|
129
128
|
|
130
129
|
def create_folder(folder)
|
131
|
-
puts
|
130
|
+
puts " create".white + " #{folder}"
|
132
131
|
FileUtils.mkdir_p folder
|
133
132
|
@folders_created += 1
|
134
133
|
end
|