aspera-cli 4.5.0 → 4.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +1 -0
- data/README.md +1894 -1574
- data/bin/ascli +21 -1
- data/bin/asession +38 -34
- data/docs/test_env.conf +14 -3
- data/examples/aoc.rb +17 -15
- data/examples/dascli +26 -0
- data/examples/faspex4.rb +42 -35
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +38 -37
- data/lib/aspera/aoc.rb +245 -205
- data/lib/aspera/ascmd.rb +111 -90
- data/lib/aspera/ats_api.rb +16 -14
- data/lib/aspera/cli/basic_auth_plugin.rb +19 -18
- data/lib/aspera/cli/extended_value.rb +50 -39
- data/lib/aspera/cli/formater.rb +161 -135
- data/lib/aspera/cli/info.rb +18 -0
- data/lib/aspera/cli/listener/line_dump.rb +4 -2
- data/lib/aspera/cli/listener/logger.rb +3 -1
- data/lib/aspera/cli/listener/progress.rb +20 -21
- data/lib/aspera/cli/listener/progress_multi.rb +29 -31
- data/lib/aspera/cli/main.rb +194 -183
- data/lib/aspera/cli/manager.rb +213 -206
- data/lib/aspera/cli/plugin.rb +71 -49
- data/lib/aspera/cli/plugins/alee.rb +8 -7
- data/lib/aspera/cli/plugins/aoc.rb +675 -558
- data/lib/aspera/cli/plugins/ats.rb +116 -109
- data/lib/aspera/cli/plugins/bss.rb +35 -34
- data/lib/aspera/cli/plugins/config.rb +722 -542
- data/lib/aspera/cli/plugins/console.rb +28 -22
- data/lib/aspera/cli/plugins/cos.rb +28 -37
- data/lib/aspera/cli/plugins/faspex.rb +281 -227
- data/lib/aspera/cli/plugins/faspex5.rb +129 -84
- data/lib/aspera/cli/plugins/node.rb +426 -232
- data/lib/aspera/cli/plugins/orchestrator.rb +106 -98
- data/lib/aspera/cli/plugins/preview.rb +196 -191
- data/lib/aspera/cli/plugins/server.rb +131 -126
- data/lib/aspera/cli/plugins/shares.rb +49 -36
- data/lib/aspera/cli/plugins/sync.rb +27 -28
- data/lib/aspera/cli/transfer_agent.rb +84 -79
- data/lib/aspera/cli/version.rb +3 -1
- data/lib/aspera/colors.rb +37 -28
- data/lib/aspera/command_line_builder.rb +84 -63
- data/lib/aspera/cos_node.rb +68 -34
- data/lib/aspera/data_repository.rb +4 -2
- data/lib/aspera/environment.rb +61 -46
- data/lib/aspera/fasp/agent_base.rb +36 -31
- data/lib/aspera/fasp/agent_connect.rb +44 -37
- data/lib/aspera/fasp/agent_direct.rb +101 -104
- data/lib/aspera/fasp/agent_httpgw.rb +91 -90
- data/lib/aspera/fasp/agent_node.rb +36 -33
- data/lib/aspera/fasp/agent_trsdk.rb +28 -31
- data/lib/aspera/fasp/error.rb +3 -1
- data/lib/aspera/fasp/error_info.rb +81 -54
- data/lib/aspera/fasp/installation.rb +171 -151
- data/lib/aspera/fasp/listener.rb +2 -0
- data/lib/aspera/fasp/parameters.rb +105 -111
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +20 -20
- data/lib/aspera/fasp/transfer_spec.rb +27 -0
- data/lib/aspera/fasp/uri.rb +31 -29
- data/lib/aspera/faspex_gw.rb +95 -118
- data/lib/aspera/hash_ext.rb +12 -13
- data/lib/aspera/id_generator.rb +11 -9
- data/lib/aspera/keychain/encrypted_hash.rb +73 -57
- data/lib/aspera/keychain/macos_security.rb +27 -29
- data/lib/aspera/log.rb +40 -39
- data/lib/aspera/nagios.rb +24 -22
- data/lib/aspera/node.rb +38 -30
- data/lib/aspera/oauth.rb +217 -248
- data/lib/aspera/open_application.rb +9 -7
- data/lib/aspera/persistency_action_once.rb +15 -14
- data/lib/aspera/persistency_folder.rb +15 -18
- data/lib/aspera/preview/file_types.rb +266 -270
- data/lib/aspera/preview/generator.rb +94 -92
- data/lib/aspera/preview/image_error.png +0 -0
- data/lib/aspera/preview/options.rb +20 -17
- data/lib/aspera/preview/utils.rb +99 -102
- data/lib/aspera/preview/video_error.png +0 -0
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +114 -21
- data/lib/aspera/rest.rb +144 -142
- data/lib/aspera/rest_call_error.rb +3 -2
- data/lib/aspera/rest_error_analyzer.rb +31 -31
- data/lib/aspera/rest_errors_aspera.rb +18 -16
- data/lib/aspera/secret_hider.rb +68 -0
- data/lib/aspera/ssh.rb +20 -16
- data/lib/aspera/sync.rb +57 -54
- data/lib/aspera/temp_file_manager.rb +20 -14
- data/lib/aspera/timer_limiter.rb +10 -8
- data/lib/aspera/uri_reader.rb +14 -15
- data/lib/aspera/web_auth.rb +85 -80
- data.tar.gz.sig +0 -0
- metadata +169 -40
- metadata.gz.sig +2 -0
- data/bin/dascli +0 -13
- data/docs/Makefile +0 -63
- data/docs/README.erb.md +0 -4221
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
@@ -1,10 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/hash_ext'
|
1
4
|
require 'openssl'
|
2
5
|
|
3
6
|
module Aspera
|
4
7
|
module Keychain
|
5
8
|
class SimpleCipher
|
6
9
|
def initialize(key)
|
7
|
-
@key=Digest::SHA1.hexdigest(key)[0..23]
|
10
|
+
@key = Digest::SHA1.hexdigest(key+('*'*23))[0..23]
|
8
11
|
@cipher = OpenSSL::Cipher.new('DES-EDE3-CBC')
|
9
12
|
end
|
10
13
|
|
@@ -12,7 +15,7 @@ module Aspera
|
|
12
15
|
@cipher.encrypt
|
13
16
|
@cipher.key = @key
|
14
17
|
s = @cipher.update(value) + @cipher.final
|
15
|
-
s.
|
18
|
+
s.unpack1('H*')
|
16
19
|
end
|
17
20
|
|
18
21
|
def decrypt(value)
|
@@ -25,93 +28,106 @@ module Aspera
|
|
25
28
|
|
26
29
|
# Manage secrets in a simple Hash
|
27
30
|
class EncryptedHash
|
28
|
-
SEPARATOR='%'
|
31
|
+
SEPARATOR = '%'
|
32
|
+
ACCEPTED_KEYS = %i[username url secret description].freeze
|
29
33
|
private_constant :SEPARATOR
|
34
|
+
attr_reader :legacy_detected
|
30
35
|
def initialize(values)
|
31
|
-
raise
|
32
|
-
@all_secrets=values
|
36
|
+
raise 'values shall be Hash' unless values.is_a?(Hash)
|
37
|
+
@all_secrets = values
|
38
|
+
end
|
39
|
+
|
40
|
+
def identifier(options)
|
41
|
+
return options[:username] if options[:url].to_s.empty?
|
42
|
+
%i[url username].map{|s|options[s]}.join(SEPARATOR)
|
33
43
|
end
|
34
44
|
|
35
45
|
def set(options)
|
36
|
-
raise
|
37
|
-
unsupported=options.keys-
|
46
|
+
raise 'options shall be Hash' unless options.is_a?(Hash)
|
47
|
+
unsupported = options.keys - ACCEPTED_KEYS
|
38
48
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
39
|
-
username=options[:username]
|
40
|
-
raise
|
41
|
-
url=options[:url]
|
42
|
-
raise
|
43
|
-
secret=options[:secret]
|
44
|
-
raise
|
45
|
-
key=
|
49
|
+
username = options[:username]
|
50
|
+
raise 'options shall have username' if username.nil?
|
51
|
+
url = options[:url]
|
52
|
+
raise 'options shall have username' if url.nil?
|
53
|
+
secret = options[:secret]
|
54
|
+
raise 'options shall have secret' if secret.nil?
|
55
|
+
key = identifier(options)
|
46
56
|
raise "secret #{key} already exist, delete first" if @all_secrets.has_key?(key)
|
47
|
-
obj={username: username, url: url, secret: SimpleCipher.new(key).encrypt(secret)}
|
48
|
-
obj[:description]=options[:description] if options.has_key?(:description)
|
49
|
-
@all_secrets[key]=obj
|
57
|
+
obj = {username: username, url: url, secret: SimpleCipher.new(key).encrypt(secret)}
|
58
|
+
obj[:description] = options[:description] if options.has_key?(:description)
|
59
|
+
@all_secrets[key] = obj.stringify_keys
|
50
60
|
nil
|
51
61
|
end
|
52
62
|
|
53
63
|
def list
|
54
|
-
result=[]
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
64
|
+
result = []
|
65
|
+
legacy_detected=false
|
66
|
+
@all_secrets.each do |name,value|
|
67
|
+
normal = # normalized version
|
68
|
+
case value
|
69
|
+
when String
|
70
|
+
legacy_detected=true
|
71
|
+
{username: name, url: '', secret: value}
|
72
|
+
when Hash then value.symbolize_keys
|
73
|
+
else raise 'error secret must be String (legacy) or Hash (new)'
|
74
|
+
end
|
75
|
+
normal[:description] = '' unless normal.has_key?(:description)
|
76
|
+
extraneous_keys=normal.keys - ACCEPTED_KEYS
|
77
|
+
Log.log.error("wrongs keys in secret hash: #{extraneous_keys.map(&:to_s).join(',')}") unless extraneous_keys.empty?
|
78
|
+
result.push(normal)
|
67
79
|
end
|
80
|
+
Log.log.warn('Legacy vault format detected in config file, please refer to documentation to convert to new format.') if legacy_detected
|
68
81
|
return result
|
69
82
|
end
|
70
83
|
|
71
84
|
def delete(options)
|
72
|
-
raise
|
73
|
-
unsupported=options.keys-[
|
85
|
+
raise 'options shall be Hash' unless options.is_a?(Hash)
|
86
|
+
unsupported = options.keys - %i[username url]
|
74
87
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
75
|
-
username=options[:username]
|
76
|
-
raise
|
77
|
-
url=options[:url]
|
78
|
-
key=nil
|
88
|
+
username = options[:username]
|
89
|
+
raise 'options shall have username' if username.nil?
|
90
|
+
url = options[:url]
|
91
|
+
key = nil
|
79
92
|
if !url.nil?
|
80
|
-
extk=
|
81
|
-
key=extk if @all_secrets.has_key?(extk)
|
93
|
+
extk = identifier(options)
|
94
|
+
key = extk if @all_secrets.has_key?(extk)
|
82
95
|
end
|
83
96
|
# backward compatibility: TODO: remove in future ? (make url mandatory ?)
|
84
|
-
key=username if key.nil?
|
85
|
-
raise
|
97
|
+
key = username if key.nil? && @all_secrets.has_key?(username)
|
98
|
+
raise 'no such secret' if key.nil?
|
86
99
|
@all_secrets.delete(key)
|
87
100
|
end
|
88
101
|
|
89
102
|
def get(options)
|
90
|
-
raise
|
91
|
-
unsupported=options.keys-[
|
103
|
+
raise 'options shall be Hash' unless options.is_a?(Hash)
|
104
|
+
unsupported = options.keys - %i[username url]
|
92
105
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
93
|
-
username=options[:username]
|
94
|
-
raise
|
95
|
-
url=options[:url]
|
96
|
-
|
106
|
+
username = options[:username]
|
107
|
+
raise 'options shall have username' if username.nil?
|
108
|
+
url = options[:url]
|
109
|
+
info = nil
|
97
110
|
if !url.nil?
|
98
|
-
|
111
|
+
info = @all_secrets[identifier(options)]
|
99
112
|
end
|
100
113
|
# backward compatibility: TODO: remove in future ? (make url mandatory ?)
|
101
|
-
if
|
102
|
-
|
114
|
+
if info.nil?
|
115
|
+
info = @all_secrets[username]
|
103
116
|
end
|
104
|
-
result=options.clone
|
105
|
-
case
|
117
|
+
result = options.clone
|
118
|
+
case info
|
106
119
|
when NilClass
|
107
|
-
raise "no such secret"
|
120
|
+
raise "no such secret: #{options[:url]} #{username}"
|
108
121
|
when String
|
109
|
-
result
|
122
|
+
result[:secret] = info
|
123
|
+
result[:description] = ''
|
110
124
|
when Hash
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
125
|
+
info=info.symbolize_keys
|
126
|
+
key = identifier(options)
|
127
|
+
plain = SimpleCipher.new(key).decrypt(info[:secret])
|
128
|
+
result[:secret] = plain
|
129
|
+
result[:description] = info[:description]
|
130
|
+
else raise 'error'
|
115
131
|
end
|
116
132
|
return result
|
117
133
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'security'
|
2
4
|
|
3
5
|
# enhance the gem to support other keychains
|
@@ -5,7 +7,7 @@ module Security
|
|
5
7
|
class Keychain
|
6
8
|
class << self
|
7
9
|
def by_name(name)
|
8
|
-
keychains_from_output(
|
10
|
+
keychains_from_output('security list-keychains').find{|kc|kc.filename.end_with?("/#{name}.keychain-db")}
|
9
11
|
end
|
10
12
|
end
|
11
13
|
end
|
@@ -15,18 +17,18 @@ module Security
|
|
15
17
|
# add some login to original method
|
16
18
|
alias_method :orig_flags_for_options, :flags_for_options
|
17
19
|
def flags_for_options(options = {})
|
18
|
-
keychain=options.delete(:keychain)
|
19
|
-
url=options.delete(:url)
|
20
|
+
keychain = options.delete(:keychain)
|
21
|
+
url = options.delete(:url)
|
20
22
|
if !url.nil?
|
21
|
-
uri=URI.parse(url)
|
23
|
+
uri = URI.parse(url)
|
22
24
|
raise 'only https' unless uri.scheme.eql?('https')
|
23
|
-
options[:r]='htps'
|
25
|
+
options[:r] = 'htps'
|
24
26
|
raise 'host required in URL' if uri.host.nil?
|
25
|
-
options[:s]=uri.host
|
26
|
-
options[:p]=uri.path unless ['','/'].include?(uri.path)
|
27
|
-
options[:P]=uri.port unless uri.port.eql?(443)
|
27
|
+
options[:s] = uri.host
|
28
|
+
options[:p] = uri.path unless ['','/'].include?(uri.path)
|
29
|
+
options[:P] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
|
28
30
|
end
|
29
|
-
flags=[orig_flags_for_options(options)]
|
31
|
+
flags = [orig_flags_for_options(options)]
|
30
32
|
flags.push(keychain.filename) unless keychain.nil?
|
31
33
|
flags.join(' ')
|
32
34
|
end
|
@@ -39,40 +41,36 @@ module Aspera
|
|
39
41
|
# keychain based on macOS keychain, using `security` cmmand line
|
40
42
|
class MacosSecurity
|
41
43
|
def initialize(name=nil)
|
42
|
-
|
43
|
-
@keychain=Security::Keychain.default_keychain
|
44
|
-
else
|
45
|
-
@keychain=Security::Keychain.by_name(name)
|
46
|
-
end
|
44
|
+
@keychain = name.nil? ? Security::Keychain.default_keychain : Security::Keychain.by_name(name)
|
47
45
|
raise "no such keychain #{name}" if @keychain.nil?
|
48
46
|
end
|
49
47
|
|
50
48
|
def set(options)
|
51
49
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
52
|
-
unsupported=options.keys-[
|
50
|
+
unsupported = options.keys - %i[username url secret description]
|
53
51
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
54
|
-
username=options[:username]
|
52
|
+
username = options[:username]
|
55
53
|
raise 'options shall have username' if username.nil?
|
56
|
-
url=options[:url]
|
54
|
+
url = options[:url]
|
57
55
|
raise 'options shall have url' if url.nil?
|
58
|
-
secret=options[:secret]
|
56
|
+
secret = options[:secret]
|
59
57
|
raise 'options shall have secret' if secret.nil?
|
60
58
|
raise 'set not implemented'
|
61
|
-
self
|
62
59
|
end
|
63
60
|
|
64
61
|
def get(options)
|
65
62
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
66
|
-
unsupported=options.keys-[
|
63
|
+
unsupported = options.keys - %i[username url]
|
67
64
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
68
|
-
username=options[:username]
|
65
|
+
username = options[:username]
|
69
66
|
raise 'options shall have username' if username.nil?
|
70
|
-
url=options[:url]
|
67
|
+
url = options[:url]
|
71
68
|
raise 'options shall have url' if url.nil?
|
72
|
-
info=Security::InternetPassword.find(keychain: @keychain, url: url, account: username)
|
69
|
+
info = Security::InternetPassword.find(keychain: @keychain, url: url, account: username)
|
73
70
|
raise 'not found' if info.nil?
|
74
|
-
result=options.clone
|
75
|
-
result
|
71
|
+
result = options.clone
|
72
|
+
result[:secret] = info.password
|
73
|
+
result[:description] = info.attributes['icmt']
|
76
74
|
return result
|
77
75
|
end
|
78
76
|
|
@@ -82,12 +80,12 @@ module Aspera
|
|
82
80
|
|
83
81
|
def delete(options)
|
84
82
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
85
|
-
unsupported=options.keys-[
|
83
|
+
unsupported = options.keys - %i[username url]
|
86
84
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
87
|
-
username=options[:username]
|
85
|
+
username = options[:username]
|
88
86
|
raise 'options shall have username' if username.nil?
|
89
|
-
url=options[:url]
|
90
|
-
raise
|
87
|
+
url = options[:url]
|
88
|
+
raise "delete not implemented #{url}"
|
91
89
|
end
|
92
90
|
end
|
93
91
|
end
|
data/lib/aspera/log.rb
CHANGED
@@ -1,4 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'aspera/colors'
|
4
|
+
require 'aspera/secret_hider'
|
2
5
|
require 'logger'
|
3
6
|
require 'pp'
|
4
7
|
require 'json'
|
@@ -11,10 +14,10 @@ module Aspera
|
|
11
14
|
# class methods
|
12
15
|
class << self
|
13
16
|
# levels are :debug,:info,:warn,:error,fatal,:unknown
|
14
|
-
def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a)<=>Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
|
17
|
+
def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
|
15
18
|
|
16
19
|
# where logs are sent to
|
17
|
-
def logtypes; [
|
20
|
+
def logtypes; %i[stderr stdout syslog];end
|
18
21
|
|
19
22
|
# get the logger object of singleton
|
20
23
|
def log; instance.logger;end
|
@@ -23,28 +26,36 @@ module Aspera
|
|
23
26
|
# @param name string or symbol
|
24
27
|
# @param format either pp or json format
|
25
28
|
def dump(name,object,format=:json)
|
26
|
-
|
27
|
-
result=
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
29
|
+
log.debug do
|
30
|
+
result =
|
31
|
+
case format
|
32
|
+
when :json
|
33
|
+
JSON.pretty_generate(object) rescue PP.pp(object,+'')
|
34
|
+
when :ruby
|
35
|
+
PP.pp(object,+'')
|
36
|
+
else
|
37
|
+
raise 'wrong parameter, expect pp or json'
|
38
|
+
end
|
35
39
|
"#{name.to_s.green} (#{format})=\n#{result}"
|
36
40
|
end
|
37
41
|
end
|
38
|
-
end
|
39
42
|
|
40
|
-
|
41
|
-
|
43
|
+
def capture_stderr
|
44
|
+
real_stderr = $stderr
|
45
|
+
$stderr = StringIO.new
|
46
|
+
yield
|
47
|
+
log.debug($stderr.string)
|
48
|
+
ensure
|
49
|
+
$stderr = real_stderr
|
50
|
+
end
|
51
|
+
end # class
|
52
|
+
|
53
|
+
attr_reader :logger_type, :logger
|
42
54
|
attr_writer :program_name
|
43
|
-
attr_accessor :log_passwords
|
44
55
|
|
45
56
|
# set log level of underlying logger given symbol level
|
46
57
|
def level=(new_level)
|
47
|
-
@logger.level=Logger::Severity.const_get(new_level.to_sym.upcase)
|
58
|
+
@logger.level = Logger::Severity.const_get(new_level.to_sym.upcase)
|
48
59
|
end
|
49
60
|
|
50
61
|
# get symbol of debug level of underlying logger
|
@@ -53,49 +64,39 @@ module Aspera
|
|
53
64
|
return name.downcase.to_sym if @logger.level.eql?(Logger::Severity.const_get(name))
|
54
65
|
end
|
55
66
|
# should not happen
|
56
|
-
raise "
|
67
|
+
raise "INTERNAL ERROR: unexpected level #{@logger.level}"
|
57
68
|
end
|
58
69
|
|
59
70
|
# change underlying logger, but keep log level
|
60
71
|
def logger_type=(new_logtype)
|
61
|
-
current_severity_integer
|
62
|
-
current_severity_integer=ENV['AS_LOG_LEVEL'] if current_severity_integer.nil?
|
63
|
-
current_severity_integer=Logger::Severity::WARN if current_severity_integer.nil?
|
72
|
+
current_severity_integer = @logger.level unless @logger.nil?
|
73
|
+
current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.has_key?('AS_LOG_LEVEL')
|
74
|
+
current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
|
64
75
|
case new_logtype
|
65
76
|
when :stderr
|
66
|
-
@logger = Logger.new(
|
77
|
+
@logger = Logger.new($stderr)
|
67
78
|
when :stdout
|
68
|
-
@logger = Logger.new(
|
79
|
+
@logger = Logger.new($stdout)
|
69
80
|
when :syslog
|
70
81
|
require 'syslog/logger'
|
71
82
|
@logger = Syslog::Logger.new(@program_name)
|
72
83
|
else
|
73
84
|
raise "unknown log type: #{new_logtype.class} #{new_logtype}"
|
74
85
|
end
|
75
|
-
@logger.level=current_severity_integer
|
76
|
-
@logger_type=new_logtype
|
77
|
-
original_formatter = @logger.formatter || Logger::Formatter.new
|
86
|
+
@logger.level = current_severity_integer
|
87
|
+
@logger_type = new_logtype
|
78
88
|
# update formatter with password hiding
|
79
|
-
@logger.formatter=
|
80
|
-
unless @log_passwords
|
81
|
-
msg=msg.gsub(/("[^"]*(password|secret|private_key)[^"]*"=>")([^"]+)(")/){"#{$1}***#{$4}"}
|
82
|
-
msg=msg.gsub(/("[^"]*(secret)[^"]*"=>{)([^}]+)(})/){"#{$1}***#{$4}"}
|
83
|
-
msg=msg.gsub(/((secrets)={)([^}]+)(})/){"#{$1}***#{$4}"}
|
84
|
-
end
|
85
|
-
original_formatter.call(severity, datetime, progname, msg)
|
86
|
-
end
|
89
|
+
@logger.formatter = SecretHider.log_formatter(@logger.formatter)
|
87
90
|
end
|
88
91
|
|
89
92
|
private
|
90
93
|
|
91
94
|
def initialize
|
92
|
-
@logger=nil
|
93
|
-
@program_name='aspera'
|
94
|
-
@log_passwords=false
|
95
|
+
@logger = nil
|
96
|
+
@program_name = 'aspera'
|
95
97
|
# this sets @logger and @logger_type (self needed to call method instead of local var)
|
96
|
-
self.logger_type
|
97
|
-
raise
|
98
|
+
self.logger_type = :stderr
|
99
|
+
raise 'error logger shall be defined' if @logger.nil?
|
98
100
|
end
|
99
|
-
|
100
101
|
end
|
101
102
|
end
|
data/lib/aspera/nagios.rb
CHANGED
@@ -1,34 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'date'
|
2
4
|
|
3
5
|
module Aspera
|
4
6
|
class Nagios
|
5
7
|
# nagios levels
|
6
|
-
LEVELS=[
|
7
|
-
ADD_PREFIX='add_'
|
8
|
+
LEVELS = %i[ok warning critical unknown dependent].freeze
|
9
|
+
ADD_PREFIX = 'add_'
|
10
|
+
# date offset levels
|
11
|
+
DATE_WARN_OFFSET = 2
|
12
|
+
DATE_CRIT_OFFSET = 5
|
13
|
+
private_constant :LEVELS,:ADD_PREFIX,:DATE_WARN_OFFSET,:DATE_CRIT_OFFSET
|
14
|
+
|
8
15
|
# add methods to add nagios error levels, each take component name and message
|
9
16
|
LEVELS.each_index do |code|
|
10
|
-
name="#{ADD_PREFIX}#{LEVELS[code]}".to_sym
|
11
|
-
define_method(name){|comp,msg|@data.push({:code
|
12
|
-
public name
|
17
|
+
name = "#{ADD_PREFIX}#{LEVELS[code]}".to_sym
|
18
|
+
define_method(name){|comp,msg|@data.push({code: code,comp: comp,msg: msg})}
|
13
19
|
end
|
14
|
-
# date offset levels
|
15
|
-
DATE_WARN_OFFSET=2
|
16
|
-
DATE_CRIT_OFFSET=5
|
17
|
-
private_constant :LEVELS,:ADD_PREFIX,:DATE_WARN_OFFSET,:DATE_CRIT_OFFSET
|
18
20
|
|
19
21
|
attr_reader :data
|
20
22
|
def initialize
|
21
|
-
@data=[]
|
23
|
+
@data = []
|
22
24
|
end
|
23
25
|
|
24
26
|
# comparte remote time with local time
|
25
|
-
def check_time_offset(
|
27
|
+
def check_time_offset(remote_date, component)
|
26
28
|
# check date if specified : 2015-10-13T07:32:01Z
|
27
29
|
rtime = DateTime.strptime(remote_date)
|
28
30
|
diff_time = (rtime - DateTime.now).abs
|
29
|
-
diff_disp=diff_time.round(-2)
|
31
|
+
diff_disp = diff_time.round(-2)
|
30
32
|
Log.log.debug("DATE: #{remote_date} #{rtime} diff=#{diff_disp}")
|
31
|
-
msg="offset #{diff_disp} sec"
|
33
|
+
msg = "offset #{diff_disp} sec"
|
32
34
|
if diff_time >= DATE_CRIT_OFFSET
|
33
35
|
add_critical(component,msg)
|
34
36
|
elsif diff_time >= DATE_WARN_OFFSET
|
@@ -38,30 +40,30 @@ module Aspera
|
|
38
40
|
end
|
39
41
|
end
|
40
42
|
|
41
|
-
def check_product_version(
|
43
|
+
def check_product_version(component, _product, version)
|
42
44
|
add_ok(component,"version #{version}")
|
43
45
|
# TODO check on database if latest version
|
44
46
|
end
|
45
47
|
|
46
48
|
# translate for display
|
47
49
|
def result
|
48
|
-
raise
|
49
|
-
{:
|
50
|
+
raise 'missing result' if @data.empty?
|
51
|
+
{type: :object_list,data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s,'component' => i[:comp],'message' => i[:msg]}}}
|
50
52
|
end
|
51
53
|
|
52
54
|
# process results of a analysis and display status and exit with code
|
53
55
|
def self.process(data)
|
54
|
-
raise
|
55
|
-
[
|
56
|
-
res_errors = data.
|
56
|
+
raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
|
57
|
+
%w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.has_key?(c)}
|
58
|
+
res_errors = data.reject{|s|s['status'].eql?('ok')}
|
57
59
|
# keep only errors in case of problem, other ok are assumed so
|
58
60
|
data = res_errors unless res_errors.empty?
|
59
61
|
# first is most critical
|
60
|
-
data.sort!{|a,b|LEVELS.index(a['status'].to_sym)<=>LEVELS.index(b['status'].to_sym)}
|
62
|
+
data.sort!{|a,b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
61
63
|
# build message: if multiple components: concatenate
|
62
64
|
#message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
63
|
-
message = data.map{|i|i['component']}.uniq.map{|comp|comp+':'+data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}.join(', ').
|
64
|
-
status=data.first['status'].upcase
|
65
|
+
message = data.map{|i|i['component']}.uniq.map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}.join(', ').tr("\n",' ')
|
66
|
+
status = data.first['status'].upcase
|
65
67
|
# display status for nagios
|
66
68
|
puts("#{status} - [#{message}]\n")
|
67
69
|
# provide exit code to nagios
|
data/lib/aspera/node.rb
CHANGED
@@ -1,7 +1,10 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/fasp/transfer_spec'
|
2
4
|
require 'aspera/rest'
|
3
5
|
require 'aspera/oauth'
|
4
6
|
require 'aspera/log'
|
7
|
+
require 'aspera/environment'
|
5
8
|
require 'zlib'
|
6
9
|
require 'base64'
|
7
10
|
|
@@ -9,32 +12,36 @@ module Aspera
|
|
9
12
|
# Provides additional functions using node API.
|
10
13
|
class Node < Rest
|
11
14
|
# permissions
|
12
|
-
ACCESS_LEVELS=[
|
13
|
-
|
15
|
+
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
16
|
+
# prefix for ruby code for filter
|
17
|
+
MATCH_EXEC_PREFIX = 'exec:'
|
14
18
|
|
15
19
|
# register node special token decoder
|
16
20
|
Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
|
17
21
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
+
class << self
|
23
|
+
def set_ak_basic_token(ts,ak,secret)
|
24
|
+
Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, "\
|
25
|
+
"but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
26
|
+
ts['token'] = Rest.basic_creds(ak,secret)
|
27
|
+
end
|
22
28
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
29
|
+
# for access keys: provide expression to match entry in folder
|
30
|
+
# if no prefix: regex
|
31
|
+
# if prefix: ruby code
|
32
|
+
# if filder is nil, then always match
|
33
|
+
def file_matcher(match_expression)
|
34
|
+
match_expression ||= "#{MATCH_EXEC_PREFIX}true"
|
35
|
+
if match_expression.start_with?(MATCH_EXEC_PREFIX)
|
36
|
+
return Environment.secure_eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}")
|
37
|
+
end
|
38
|
+
return lambda{|f|f['name'].match(/#{match_expression}/)}
|
31
39
|
end
|
32
|
-
return lambda{|f|f['name'].match(/#{match_expression}/)}
|
33
40
|
end
|
34
41
|
|
35
|
-
def initialize(rest_params)
|
36
|
-
super(rest_params)
|
37
|
-
end
|
42
|
+
# def initialize(rest_params)
|
43
|
+
# super(rest_params)
|
44
|
+
# end
|
38
45
|
|
39
46
|
# recursively crawl in a folder.
|
40
47
|
# subfolders a processed if the processing method returns true
|
@@ -52,25 +59,26 @@ module Aspera
|
|
52
59
|
raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
|
53
60
|
Log.log.debug("crawl #{opt}")
|
54
61
|
#top_info=read("files/#{opt[:top_file_id]}")[:data]
|
55
|
-
folders_to_explore=[{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
|
62
|
+
folders_to_explore = [{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
|
56
63
|
Log.dump(:folders_to_explore,folders_to_explore)
|
57
|
-
while !folders_to_explore.empty?
|
64
|
+
while !folders_to_explore.empty?
|
58
65
|
current_item = folders_to_explore.shift
|
59
66
|
Log.log.debug("searching #{current_item[:relpath]}".bg_green)
|
60
67
|
# get folder content
|
61
|
-
folder_contents =
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
68
|
+
folder_contents =
|
69
|
+
begin
|
70
|
+
read("files/#{current_item[:id]}/files")[:data]
|
71
|
+
rescue StandardError => e
|
72
|
+
Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
|
73
|
+
[]
|
74
|
+
end
|
67
75
|
Log.dump(:folder_contents,folder_contents)
|
68
76
|
folder_contents.each do |entry|
|
69
|
-
relative_path=File.join(current_item[:relpath],entry['name'])
|
77
|
+
relative_path = File.join(current_item[:relpath],entry['name'])
|
70
78
|
Log.log.debug("looking #{relative_path}".bg_green)
|
71
79
|
# entry type is file, folder or link
|
72
|
-
if processor.send(opt[:method],entry,relative_path)
|
73
|
-
folders_to_explore.push({:
|
80
|
+
if processor.send(opt[:method],entry,relative_path) && entry['type'].eql?('folder')
|
81
|
+
folders_to_explore.push({id: entry['id'],relpath: relative_path})
|
74
82
|
end
|
75
83
|
end
|
76
84
|
end
|