nsrr 0.4.0.beta1 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +19 -1
- data/LICENSE +20 -111
- data/README.md +14 -52
- data/Rakefile +4 -4
- data/bin/nsrr +1 -1
- data/lib/nsrr.rb +11 -11
- data/lib/nsrr/commands/console.rb +3 -3
- data/lib/nsrr/commands/download.rb +39 -20
- data/lib/nsrr/commands/update.rb +8 -8
- data/lib/nsrr/helpers/authorization.rb +9 -8
- data/lib/nsrr/helpers/color.rb +100 -0
- data/lib/nsrr/helpers/constants.rb +1 -1
- data/lib/nsrr/helpers/download_request.rb +9 -12
- data/lib/nsrr/helpers/hash_helper.rb +4 -1
- data/lib/nsrr/helpers/json_request.rb +32 -19
- data/lib/nsrr/models/all.rb +1 -1
- data/lib/nsrr/models/dataset.rb +36 -37
- data/lib/nsrr/models/file.rb +34 -33
- data/lib/nsrr/version.rb +4 -4
- data/nsrr.gemspec +18 -15
- metadata +15 -29
data/lib/nsrr/commands/update.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require "nsrr/helpers/color"
|
4
|
+
require "nsrr/helpers/json_request"
|
5
5
|
|
6
6
|
module Nsrr
|
7
7
|
module Commands
|
@@ -17,19 +17,19 @@ module Nsrr
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def start
|
20
|
-
(json, _status) = Nsrr::Helpers::JsonRequest.get(
|
20
|
+
(json, _status) = Nsrr::Helpers::JsonRequest.get("https://rubygems.org/api/v1/gems/nsrr.json")
|
21
21
|
if json
|
22
|
-
if json[
|
23
|
-
puts
|
22
|
+
if json["version"] == Nsrr::VERSION::STRING
|
23
|
+
puts "The nsrr gem is " + "up-to-date".green + "!"
|
24
24
|
else
|
25
25
|
puts
|
26
|
-
puts "A newer version (v#{json[
|
26
|
+
puts "A newer version (v#{json["version"]}) is available! Type the following command to update:"
|
27
27
|
puts
|
28
|
-
puts
|
28
|
+
puts " gem install nsrr --no-document".white
|
29
29
|
puts
|
30
30
|
end
|
31
31
|
else
|
32
|
-
puts
|
32
|
+
puts "Unable to connect to RubyGems.org. Please try again later."
|
33
33
|
end
|
34
34
|
end
|
35
35
|
end
|
@@ -1,22 +1,23 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "nsrr/helpers/color"
|
4
|
+
require "nsrr/helpers/json_request"
|
4
5
|
|
5
6
|
module Nsrr
|
6
7
|
module Helpers
|
7
8
|
# Helper to verify that user is authenticated.
|
8
9
|
class Authorization
|
9
10
|
def self.get_token(token)
|
10
|
-
puts
|
11
|
-
puts
|
12
|
-
print
|
13
|
-
token = STDIN.noecho(&:gets).chomp if token.to_s.strip ==
|
11
|
+
puts " Get your token here: " + "#{Nsrr::WEBSITE}/token".bg_gray.blue.underline
|
12
|
+
puts " Your input is hidden while entering token.".white
|
13
|
+
print " Enter your token: "
|
14
|
+
token = STDIN.noecho(&:gets).chomp if token.to_s.strip == ""
|
14
15
|
token.strip!
|
15
16
|
(response, _status) = Nsrr::Helpers::JsonRequest.get("#{Nsrr::WEBSITE}/api/v1/account/profile.json", auth_token: token)
|
16
|
-
if response.is_a?(Hash) && response[
|
17
|
-
puts
|
17
|
+
if response.is_a?(Hash) && response["authenticated"]
|
18
|
+
puts "AUTHORIZED".green + " as " + "#{response["first_name"]} #{response["last_name"]}".white
|
18
19
|
else
|
19
|
-
puts
|
20
|
+
puts "UNAUTHORIZED".red + " Public Access Only"
|
20
21
|
end
|
21
22
|
token
|
22
23
|
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Adds ANSI colors to string class.
|
4
|
+
class String
|
5
|
+
def black
|
6
|
+
"\e[30m#{self}\e[0m"
|
7
|
+
end
|
8
|
+
|
9
|
+
def red
|
10
|
+
"\e[31m#{self}\e[0m"
|
11
|
+
end
|
12
|
+
|
13
|
+
def green
|
14
|
+
"\e[32m#{self}\e[0m"
|
15
|
+
end
|
16
|
+
|
17
|
+
def brown
|
18
|
+
"\e[33m#{self}\e[0m"
|
19
|
+
end
|
20
|
+
|
21
|
+
def yellow
|
22
|
+
brown
|
23
|
+
end
|
24
|
+
|
25
|
+
def blue
|
26
|
+
"\e[34m#{self}\e[0m"
|
27
|
+
end
|
28
|
+
|
29
|
+
def magenta
|
30
|
+
"\e[35m#{self}\e[0m"
|
31
|
+
end
|
32
|
+
|
33
|
+
def cyan
|
34
|
+
"\e[36m#{self}\e[0m"
|
35
|
+
end
|
36
|
+
|
37
|
+
def gray
|
38
|
+
"\e[37m#{self}\e[0m"
|
39
|
+
end
|
40
|
+
|
41
|
+
def white
|
42
|
+
"\e[39m#{bold}\e[0m"
|
43
|
+
end
|
44
|
+
|
45
|
+
def bg_black
|
46
|
+
"\e[40m#{self}\e[0m"
|
47
|
+
end
|
48
|
+
|
49
|
+
def bg_red
|
50
|
+
"\e[41m#{self}\e[0m"
|
51
|
+
end
|
52
|
+
|
53
|
+
def bg_green
|
54
|
+
"\e[42m#{self}\e[0m"
|
55
|
+
end
|
56
|
+
|
57
|
+
def bg_brown
|
58
|
+
"\e[43m#{self}\e[0m"
|
59
|
+
end
|
60
|
+
|
61
|
+
def bg_blue
|
62
|
+
"\e[44m#{self}\e[0m"
|
63
|
+
end
|
64
|
+
|
65
|
+
def bg_magenta
|
66
|
+
"\e[45m#{self}\e[0m"
|
67
|
+
end
|
68
|
+
|
69
|
+
def bg_cyan
|
70
|
+
"\e[46m#{self}\e[0m"
|
71
|
+
end
|
72
|
+
|
73
|
+
def bg_gray
|
74
|
+
"\e[47m#{self}\e[0m"
|
75
|
+
end
|
76
|
+
|
77
|
+
def bold
|
78
|
+
"\e[1m#{self}\e[22m"
|
79
|
+
end
|
80
|
+
|
81
|
+
def italic
|
82
|
+
"\e[3m#{self}\e[23m"
|
83
|
+
end
|
84
|
+
|
85
|
+
def underline
|
86
|
+
"\e[4m#{self}\e[24m"
|
87
|
+
end
|
88
|
+
|
89
|
+
def blink
|
90
|
+
"\e[5m#{self}\e[25m"
|
91
|
+
end
|
92
|
+
|
93
|
+
def reverse_color
|
94
|
+
"\e[7m#{self}\e[27m"
|
95
|
+
end
|
96
|
+
|
97
|
+
def colorless
|
98
|
+
gsub(/\e\[\d{1,2}m/, "")
|
99
|
+
end
|
100
|
+
end
|
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "openssl"
|
4
|
+
require "net/http"
|
5
|
+
require "uri"
|
6
6
|
|
7
7
|
module Nsrr
|
8
8
|
module Helpers
|
@@ -17,34 +17,31 @@ module Nsrr
|
|
17
17
|
attr_reader :url, :error, :file_size
|
18
18
|
|
19
19
|
def initialize(url, download_folder)
|
20
|
-
|
21
|
-
@url = URI.parse(escaped_url)
|
20
|
+
@url = URI.parse(url)
|
22
21
|
@http = Net::HTTP.new(@url.host, @url.port)
|
23
|
-
if @url.scheme ==
|
22
|
+
if @url.scheme == "https"
|
24
23
|
@http.use_ssl = true
|
25
24
|
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
26
25
|
end
|
27
26
|
@download_folder = download_folder
|
28
27
|
@file_size = 0
|
29
|
-
rescue
|
30
|
-
@error = 'Invalid Token'
|
31
28
|
end
|
32
29
|
|
33
30
|
# Writes file segments to disk immediately instead of storing in memory
|
34
31
|
def get
|
35
32
|
return unless @error.nil?
|
36
|
-
local_file = ::File.open(@download_folder,
|
33
|
+
local_file = ::File.open(@download_folder, "wb")
|
37
34
|
partial = true
|
38
35
|
@http.request_get(@url.path) do |response|
|
39
36
|
case response.code
|
40
|
-
when
|
37
|
+
when "200"
|
41
38
|
response.read_body do |segment|
|
42
39
|
local_file.write(segment)
|
43
40
|
end
|
44
41
|
@file_size = ::File.size(@download_folder)
|
45
42
|
partial = false
|
46
|
-
when
|
47
|
-
@error =
|
43
|
+
when "302"
|
44
|
+
@error = "Token Not Authorized to Access Specified File"
|
48
45
|
else
|
49
46
|
@error = "#{response.code} #{response.class.name}"
|
50
47
|
end
|
@@ -5,7 +5,10 @@ module Nsrr
|
|
5
5
|
# Converts hash with string keys into hash with keys as symbols.
|
6
6
|
class HashHelper
|
7
7
|
def self.symbolize_keys(hash)
|
8
|
-
hash.inject({})
|
8
|
+
hash.inject({}) do |memo, (k, v)|
|
9
|
+
memo[k.to_sym] = v
|
10
|
+
memo
|
11
|
+
end
|
9
12
|
end
|
10
13
|
end
|
11
14
|
end
|
@@ -1,9 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
3
|
+
require "openssl"
|
4
|
+
require "net/http"
|
5
|
+
require "json"
|
6
|
+
require "cgi"
|
7
|
+
|
8
|
+
require "nsrr/helpers/color"
|
7
9
|
|
8
10
|
module Nsrr
|
9
11
|
module Helpers
|
@@ -30,41 +32,52 @@ module Nsrr
|
|
30
32
|
@url = URI.parse(url)
|
31
33
|
|
32
34
|
@http = Net::HTTP.new(@url.host, @url.port)
|
33
|
-
if @url.scheme ==
|
35
|
+
if @url.scheme == "https"
|
34
36
|
@http.use_ssl = true
|
35
37
|
@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
36
38
|
end
|
37
39
|
rescue
|
38
40
|
@error = "Invalid URL: #{url.inspect}"
|
39
|
-
puts @error.
|
41
|
+
puts @error.red
|
40
42
|
end
|
41
43
|
|
42
44
|
def get
|
43
45
|
return unless @error.nil?
|
46
|
+
|
44
47
|
full_path = @url.path
|
45
|
-
query = ([@url.query] + @params).flatten.compact.join(
|
46
|
-
full_path += "?#{query}" if query.to_s !=
|
48
|
+
query = ([@url.query] + @params).flatten.compact.join("&")
|
49
|
+
full_path += "?#{query}" if query.to_s != ""
|
47
50
|
response = @http.start do |http|
|
48
51
|
http.get(full_path)
|
49
52
|
end
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
+
if response.body
|
54
|
+
[JSON.parse(response.body), response]
|
55
|
+
else
|
56
|
+
[nil, response]
|
57
|
+
end
|
58
|
+
rescue => e # JSON::ParserError, Net::ReadTimeout
|
59
|
+
puts "GET error: #{e}".red
|
60
|
+
[nil, nil]
|
53
61
|
end
|
54
62
|
|
55
63
|
def post
|
56
64
|
return unless @error.nil?
|
65
|
+
|
57
66
|
response = @http.start do |http|
|
58
|
-
http.post(@url.path, @params.flatten.compact.join(
|
67
|
+
http.post(@url.path, @params.flatten.compact.join("&"))
|
68
|
+
end
|
69
|
+
if response.body
|
70
|
+
[JSON.parse(response.body), response]
|
71
|
+
else
|
72
|
+
[nil, response]
|
59
73
|
end
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
nil
|
74
|
+
rescue => e # JSON::ParserError, Net::ReadTimeout
|
75
|
+
puts "POST error: #{e}".red
|
76
|
+
[nil, nil]
|
64
77
|
end
|
65
78
|
|
66
79
|
def patch
|
67
|
-
@params <<
|
80
|
+
@params << "_method=patch"
|
68
81
|
post
|
69
82
|
end
|
70
83
|
|
@@ -79,10 +92,10 @@ module Nsrr
|
|
79
92
|
if value.is_a? Hash
|
80
93
|
value.collect do |k,v|
|
81
94
|
key_value_to_string(k, v, current_scope)
|
82
|
-
end.join(
|
95
|
+
end.join("&")
|
83
96
|
elsif value.is_a? Array
|
84
97
|
value.collect do |v|
|
85
|
-
key_value_to_string(
|
98
|
+
key_value_to_string("", v, current_scope)
|
86
99
|
end
|
87
100
|
else
|
88
101
|
"#{current_scope}=#{CGI.escape(value.to_s)}"
|
data/lib/nsrr/models/all.rb
CHANGED
data/lib/nsrr/models/dataset.rb
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require 'io/console'
|
3
|
+
require "fileutils"
|
4
|
+
require "irb"
|
5
|
+
require "io/console"
|
7
6
|
|
8
|
-
require
|
9
|
-
require
|
10
|
-
require
|
11
|
-
require
|
7
|
+
require "nsrr/helpers/constants"
|
8
|
+
require "nsrr/helpers/color"
|
9
|
+
require "nsrr/helpers/hash_helper"
|
10
|
+
require "nsrr/helpers/json_request"
|
11
|
+
require "nsrr/helpers/authorization"
|
12
12
|
|
13
|
-
require
|
13
|
+
require "nsrr/models/file"
|
14
14
|
|
15
15
|
module Nsrr
|
16
16
|
module Models
|
@@ -18,20 +18,19 @@ module Nsrr
|
|
18
18
|
# allowing dataset files to be downloaded.
|
19
19
|
class Dataset
|
20
20
|
def self.find(slug, token = nil)
|
21
|
-
(json, _status) = Nsrr::Helpers::JsonRequest.get(
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
end
|
21
|
+
(json, _status) = Nsrr::Helpers::JsonRequest.get(
|
22
|
+
"#{Nsrr::WEBSITE}/api/v1/datasets/#{slug}.json",
|
23
|
+
auth_token: token
|
24
|
+
)
|
25
|
+
new(json, token) if json
|
27
26
|
end
|
28
27
|
|
29
28
|
attr_accessor :download_token
|
30
29
|
attr_reader :slug, :name
|
31
30
|
|
32
31
|
def initialize(json = {}, token = nil)
|
33
|
-
@slug = json[
|
34
|
-
@name = json[
|
32
|
+
@slug = json["slug"]
|
33
|
+
@name = json["name"]
|
35
34
|
@files = {}
|
36
35
|
@download_token = token
|
37
36
|
@downloaded_folders = []
|
@@ -52,16 +51,16 @@ module Nsrr
|
|
52
51
|
|
53
52
|
# Options include:
|
54
53
|
# method:
|
55
|
-
#
|
56
|
-
#
|
57
|
-
#
|
54
|
+
# "md5" => [default] Checks if a downloaded file exists with the exact md5 as the online version, if so, skips that file
|
55
|
+
# "fresh" => Downloads every file without checking if it was already downloaded
|
56
|
+
# "fast" => Only checks if a download file exists with the same file size as the online version, if so, skips that file
|
58
57
|
# depth:
|
59
|
-
#
|
60
|
-
#
|
58
|
+
# "recursive" => [default] Downloads files in selected path folder and all subfolders
|
59
|
+
# "shallow" => Only downloads files in selected path folder
|
61
60
|
def download(full_path = nil, *args)
|
62
61
|
options = Nsrr::Helpers::HashHelper.symbolize_keys(args.first || {})
|
63
|
-
options[:method] ||=
|
64
|
-
options[:depth] ||=
|
62
|
+
options[:method] ||= "md5"
|
63
|
+
options[:depth] ||= "recursive"
|
65
64
|
@folders_created = 0
|
66
65
|
@files_downloaded = 0
|
67
66
|
@downloaded_bytes = 0
|
@@ -69,9 +68,9 @@ module Nsrr
|
|
69
68
|
@files_failed = 0
|
70
69
|
|
71
70
|
begin
|
72
|
-
puts
|
73
|
-
puts
|
74
|
-
puts
|
71
|
+
puts " File Check: " + options[:method].to_s.white
|
72
|
+
puts " Depth: " + options[:depth].to_s.white
|
73
|
+
puts ""
|
75
74
|
if @download_token.nil?
|
76
75
|
@download_token = Nsrr::Helpers::Authorization.get_token(@download_token)
|
77
76
|
end
|
@@ -80,17 +79,17 @@ module Nsrr
|
|
80
79
|
|
81
80
|
download_helper(full_path, options)
|
82
81
|
rescue Interrupt, IRB::Abort
|
83
|
-
puts "\n Interrupted".
|
82
|
+
puts "\n Interrupted".red
|
84
83
|
end
|
85
84
|
|
86
85
|
@downloaded_megabytes = @downloaded_bytes / (1024 * 1024)
|
87
86
|
|
88
87
|
puts "\nFinished in #{Time.now - @start_time} seconds." if @start_time
|
89
|
-
puts "\n#{@folders_created} folder#{
|
90
|
-
"#{@files_downloaded} file#{
|
91
|
-
"#{@downloaded_megabytes} MiB#{
|
92
|
-
"#{@files_skipped} file#{
|
93
|
-
"#{@files_failed} file#{
|
88
|
+
puts "\n#{@folders_created} folder#{"s" if @folders_created != 1} created".white + ", " +
|
89
|
+
"#{@files_downloaded} file#{"s" if @files_downloaded != 1} downloaded".green + ", " +
|
90
|
+
"#{@downloaded_megabytes} MiB#{"s" if @downloaded_megabytes != 1} downloaded".green + ", " +
|
91
|
+
"#{@files_skipped} file#{"s" if @files_skipped != 1} skipped".blue + ", " +
|
92
|
+
"#{@files_failed} file#{"s" if @files_failed != 1} failed".send(@files_failed.zero? ? :white : :red) + "\n\n"
|
94
93
|
nil
|
95
94
|
end
|
96
95
|
|
@@ -104,16 +103,16 @@ module Nsrr
|
|
104
103
|
current_folder = ::File.join(slug.to_s, file.folder.to_s)
|
105
104
|
result = file.download(options[:method], current_folder, @download_token)
|
106
105
|
case result
|
107
|
-
when
|
106
|
+
when "fail"
|
108
107
|
@files_failed += 1
|
109
|
-
when
|
108
|
+
when "skip"
|
110
109
|
@files_skipped += 1
|
111
110
|
else
|
112
111
|
@files_downloaded += 1
|
113
112
|
@downloaded_bytes += result
|
114
113
|
end
|
115
114
|
end
|
116
|
-
if options[:depth] ==
|
115
|
+
if options[:depth] == "recursive"
|
117
116
|
files(full_path).reject(&:is_file).each do |file|
|
118
117
|
download_helper(file.full_path, options)
|
119
118
|
end
|
@@ -128,7 +127,7 @@ module Nsrr
|
|
128
127
|
end
|
129
128
|
|
130
129
|
def create_folder(folder)
|
131
|
-
puts
|
130
|
+
puts " create".white + " #{folder}"
|
132
131
|
FileUtils.mkdir_p folder
|
133
132
|
@folders_created += 1
|
134
133
|
end
|