aspera-cli 4.9.0 → 4.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +20 -0
- data/CHANGELOG.md +509 -0
- data/CONTRIBUTING.md +118 -0
- data/README.md +1241 -916
- data/bin/ascli +4 -4
- data/bin/asession +11 -11
- data/docs/test_env.conf +32 -21
- data/examples/aoc.rb +4 -4
- data/examples/dascli +16 -9
- data/examples/faspex4.rb +8 -8
- data/examples/node.rb +12 -12
- data/examples/server.rb +10 -10
- data/lib/aspera/aoc.rb +273 -266
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formater.rb +64 -64
- data/lib/aspera/cli/info.rb +2 -2
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +14 -19
- data/lib/aspera/cli/main.rb +66 -67
- data/lib/aspera/cli/manager.rb +112 -110
- data/lib/aspera/cli/plugin.rb +57 -36
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +309 -670
- data/lib/aspera/cli/plugins/ats.rb +44 -46
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +497 -378
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +112 -114
- data/lib/aspera/cli/plugins/faspex5.rb +71 -46
- data/lib/aspera/cli/plugins/node.rb +379 -283
- data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
- data/lib/aspera/cli/plugins/preview.rb +122 -114
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +30 -29
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +60 -59
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +27 -27
- data/lib/aspera/cos_node.rb +22 -20
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +35 -15
- data/lib/aspera/fasp/agent_base.rb +15 -15
- data/lib/aspera/fasp/agent_connect.rb +23 -21
- data/lib/aspera/fasp/agent_direct.rb +66 -64
- data/lib/aspera/fasp/agent_httpgw.rb +141 -78
- data/lib/aspera/fasp/agent_node.rb +23 -21
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +79 -79
- data/lib/aspera/fasp/listener.rb +1 -1
- data/lib/aspera/fasp/parameters.rb +86 -71
- data/lib/aspera/fasp/parameters.yaml +7 -4
- data/lib/aspera/fasp/resume_policy.rb +8 -8
- data/lib/aspera/fasp/transfer_spec.rb +35 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +38 -105
- data/lib/aspera/keychain/macos_security.rb +128 -57
- data/lib/aspera/log.rb +7 -7
- data/lib/aspera/nagios.rb +19 -18
- data/lib/aspera/node.rb +209 -35
- data/lib/aspera/oauth.rb +37 -36
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +16 -15
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +41 -41
- data/lib/aspera/proxy_auto_config.rb +21 -14
- data/lib/aspera/rest.rb +72 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +18 -17
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +15 -13
- data/lib/aspera/ssh.rb +11 -10
- data/lib/aspera/sync.rb +158 -44
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +14 -13
- data.tar.gz.sig +0 -0
- metadata +11 -36
- metadata.gz.sig +0 -0
@@ -20,13 +20,13 @@ module Aspera
|
|
20
20
|
@parameters = DEFAULTS.dup
|
21
21
|
if !params.nil?
|
22
22
|
raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
|
23
|
-
params.each do |k,v|
|
24
|
-
raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.
|
23
|
+
params.each do |k, v|
|
24
|
+
raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.key?(k)
|
25
25
|
raise "#{k} must be Integer" unless v.is_a?(Integer)
|
26
26
|
@parameters[k] = v
|
27
27
|
end
|
28
28
|
end
|
29
|
-
Log.log.debug
|
29
|
+
Log.log.debug{"resume params=#{@parameters}"}
|
30
30
|
end
|
31
31
|
|
32
32
|
# calls block a number of times (resumes) until success or limit reached
|
@@ -36,20 +36,20 @@ module Aspera
|
|
36
36
|
# maximum of retry
|
37
37
|
remaining_resumes = @parameters[:iter_max]
|
38
38
|
sleep_seconds = @parameters[:sleep_initial]
|
39
|
-
Log.log.debug
|
39
|
+
Log.log.debug{"retries=#{remaining_resumes}"}
|
40
40
|
# try to send the file until ascp is succesful
|
41
41
|
loop do
|
42
|
-
Log.log.debug('transfer starting')
|
42
|
+
Log.log.debug('transfer starting')
|
43
43
|
begin
|
44
44
|
# call provided block
|
45
45
|
yield
|
46
46
|
break
|
47
47
|
rescue Fasp::Error => e
|
48
|
-
Log.log.warn
|
48
|
+
Log.log.warn{"An error occurred: #{e.message}"}
|
49
49
|
# failure in ascp
|
50
50
|
if e.retryable?
|
51
51
|
# exit if we exceed the max number of retry
|
52
|
-
raise Fasp::Error,'Maximum number of retry reached' if remaining_resumes <= 0
|
52
|
+
raise Fasp::Error, 'Maximum number of retry reached' if remaining_resumes <= 0
|
53
53
|
else
|
54
54
|
# give one chance only to non retryable errors
|
55
55
|
unless remaining_resumes.eql?(@parameters[:iter_max])
|
@@ -61,7 +61,7 @@ module Aspera
|
|
61
61
|
|
62
62
|
# take this retry in account
|
63
63
|
remaining_resumes -= 1
|
64
|
-
Log.log.warn
|
64
|
+
Log.log.warn{"resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
|
65
65
|
|
66
66
|
# wait a bit before retrying, maybe network condition will be better
|
67
67
|
sleep(sleep_seconds)
|
@@ -16,10 +16,43 @@ module Aspera
|
|
16
16
|
'fasp_port' => UDP_PORT
|
17
17
|
}.freeze
|
18
18
|
# define constants for enums of parameters: <paramater>_<enum>, e.g. CIPHER_AES_128
|
19
|
-
Aspera::Fasp::Parameters.description.each do |k,v|
|
19
|
+
Aspera::Fasp::Parameters.description.each do |k, v|
|
20
20
|
next unless v[:enum].is_a?(Array)
|
21
21
|
v[:enum].each do |enum|
|
22
|
-
TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/,'_')}", enum.freeze)
|
22
|
+
TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
class << self
|
26
|
+
def ascp_opts_to_ts(tspec, opts)
|
27
|
+
return if opts.nil?
|
28
|
+
raise 'ascp options must be an Array' unless opts.is_a?(Array)
|
29
|
+
raise 'transfer spec must be a Hash' unless tspec.is_a?(Hash)
|
30
|
+
raise 'ascp options must be an Array or Strings' if opts.any?{|o|!o.is_a?(String)}
|
31
|
+
tspec['EX_ascp_args'] ||= []
|
32
|
+
raise 'EX_ascp_args must be an Array' unless tspec['EX_ascp_args'].is_a?(Array)
|
33
|
+
# TODO: translate command line args into transfer spec
|
34
|
+
# non translatable args are left in special ts parameter
|
35
|
+
tspec['EX_ascp_args'] = tspec['EX_ascp_args'].concat(opts)
|
36
|
+
return tspec
|
37
|
+
end
|
38
|
+
|
39
|
+
def action_to_direction(tspec, command)
|
40
|
+
raise 'transfer spec must be a Hash' unless tspec.is_a?(Hash)
|
41
|
+
tspec['direction'] = case command.to_sym
|
42
|
+
when :upload then DIRECTION_SEND
|
43
|
+
when :download then DIRECTION_RECEIVE
|
44
|
+
else raise 'Error: upload or download only'
|
45
|
+
end
|
46
|
+
return tspec
|
47
|
+
end
|
48
|
+
|
49
|
+
def action(tspec)
|
50
|
+
raise 'transfer spec must be a Hash' unless tspec.is_a?(Hash)
|
51
|
+
return case tspec['direction']
|
52
|
+
when DIRECTION_SEND then :upload
|
53
|
+
when DIRECTION_RECEIVE then :download
|
54
|
+
else raise 'Error: upload or download only'
|
55
|
+
end
|
23
56
|
end
|
24
57
|
end
|
25
58
|
end
|
data/lib/aspera/fasp/uri.rb
CHANGED
@@ -5,10 +5,10 @@ require 'aspera/command_line_builder'
|
|
5
5
|
|
6
6
|
module Aspera
|
7
7
|
module Fasp
|
8
|
-
# translates a "faspe:" URI (used in Faspex) into transfer spec hash
|
8
|
+
# translates a "faspe:" URI (used in Faspex 4) into transfer spec hash
|
9
9
|
class Uri
|
10
10
|
def initialize(fasplink)
|
11
|
-
@fasp_uri = URI.parse(fasplink.gsub(' ','%20'))
|
11
|
+
@fasp_uri = URI.parse(fasplink.gsub(' ', '%20'))
|
12
12
|
# TODO: check scheme is faspe
|
13
13
|
end
|
14
14
|
|
@@ -34,16 +34,16 @@ module Aspera
|
|
34
34
|
when 'minrate' then result_ts['min_rate_kbps'] = value.to_i
|
35
35
|
when 'port' then result_ts['fasp_port'] = value.to_i
|
36
36
|
when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
|
37
|
-
when 'enc' then result_ts['cipher'] = value.gsub(/^aes/,'aes-').gsub(/cfb$/,'-cfb').gsub(/gcm$/,'-gcm').gsub(/--/,'-')
|
37
|
+
when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub(/--/, '-')
|
38
38
|
when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
|
39
39
|
when 'createpath' then result_ts['create_dir'] = CommandLineBuilder.yes_to_true(value)
|
40
40
|
when 'fallback' then result_ts['http_fallback'] = CommandLineBuilder.yes_to_true(value)
|
41
41
|
when 'lockpolicy' then result_ts['lock_rate_policy'] = CommandLineBuilder.yes_to_true(value)
|
42
42
|
when 'lockminrate' then result_ts['lock_min_rate'] = CommandLineBuilder.yes_to_true(value)
|
43
|
-
when 'auth' then Log.log.debug
|
44
|
-
when 'v' then Log.log.debug
|
45
|
-
when 'protect' then Log.log.debug
|
46
|
-
else Log.log.warn
|
43
|
+
when 'auth' then Log.log.debug{"ignoring auth #{name}=#{value}"} # TODO: translate into transfer spec ? yes/no
|
44
|
+
when 'v' then Log.log.debug{"ignoring v #{name}=#{value}"} # TODO: translate into transfer spec ? 2
|
45
|
+
when 'protect' then Log.log.debug{"ignoring protect #{name}=#{value}"} # TODO: translate into transfer spec ?
|
46
|
+
else Log.log.warn{"URI parameter ignored: #{name} = #{value}"}
|
47
47
|
end
|
48
48
|
end
|
49
49
|
return result_ts
|
data/lib/aspera/faspex_gw.rb
CHANGED
@@ -37,12 +37,13 @@ module Aspera
|
|
37
37
|
|
38
38
|
faspex_pkg_parameters = JSON.parse(request.body)
|
39
39
|
faspex_pkg_delivery = faspex_pkg_parameters['delivery']
|
40
|
-
Log.log.debug
|
40
|
+
Log.log.debug{"faspex pkg create parameters=#{faspex_pkg_parameters}"}
|
41
41
|
|
42
42
|
# get recipient ids
|
43
43
|
files_pkg_recipients = []
|
44
44
|
faspex_pkg_delivery['recipients'].each do |recipient_email|
|
45
|
-
user_lookup = @aoc_api_user.read(
|
45
|
+
user_lookup = @aoc_api_user.read(
|
46
|
+
'contacts',
|
46
47
|
{ 'current_workspace_id' => @aoc_workspace_id, 'q' => recipient_email })[:data]
|
47
48
|
raise StandardError,
|
48
49
|
"no such unique user: #{recipient_email} / #{user_lookup}" unless !user_lookup.nil? && user_lookup.length.eql?(1)
|
@@ -67,7 +68,8 @@ module Aspera
|
|
67
68
|
node_info = @aoc_api_user.read("nodes/#{the_package['node_id']}")[:data]
|
68
69
|
|
69
70
|
# get transfer token (for node)
|
70
|
-
node_auth_bearer_token = @aoc_api_user.oauth_token(scope: AoC.node_scope(
|
71
|
+
node_auth_bearer_token = @aoc_api_user.oauth_token(scope: AoC.node_scope(
|
72
|
+
node_info['access_key'],
|
71
73
|
AoC::SCOPE_NODE_USER))
|
72
74
|
|
73
75
|
# tell Files what to expect in package: 1 transfer (can also be done after transfer)
|
@@ -123,7 +125,7 @@ module Aspera
|
|
123
125
|
'links' => { 'status' => 'unused' },
|
124
126
|
'xfer_sessions' => [faspex_transfer_spec]
|
125
127
|
}
|
126
|
-
Log.log.info
|
128
|
+
Log.log.info{"faspex_package_create_result=#{faspex_package_create_result}"}
|
127
129
|
response.status = 200
|
128
130
|
response.content_type = 'application/json'
|
129
131
|
response.body = JSON.generate(faspex_package_create_result)
|
@@ -160,7 +162,7 @@ module Aspera
|
|
160
162
|
SSLEnable: true,
|
161
163
|
SSLCertName: [['CN', WEBrick::Utils.getservername]]
|
162
164
|
}
|
163
|
-
Log.log.info
|
165
|
+
Log.log.info{"Server started on port #{webrick_options[:Port]}"}
|
164
166
|
@server = WEBrick::HTTPServer.new(webrick_options)
|
165
167
|
@server.mount('/aspera/faspex', FxGwServlet, a_aoc_api_user, a_workspace_id)
|
166
168
|
@server.mount('/newuser', NewUserServlet)
|
data/lib/aspera/hash_ext.rb
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
class ::Hash
|
4
4
|
def deep_merge(second)
|
5
|
-
merge(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
|
5
|
+
merge(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
|
6
6
|
end
|
7
7
|
|
8
8
|
def deep_merge!(second)
|
9
|
-
merge!(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
|
9
|
+
merge!(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
|
10
10
|
end
|
11
11
|
end
|
12
12
|
|
@@ -14,7 +14,7 @@ end
|
|
14
14
|
unless Hash.method_defined?(:transform_keys)
|
15
15
|
class Hash
|
16
16
|
def transform_keys
|
17
|
-
return each_with_object({}){|(k,v),memo|memo[yield(k)]=v} if block_given?
|
17
|
+
return each_with_object({}){|(k, v), memo|memo[yield(k)] = v} if block_given?
|
18
18
|
raise 'missing block'
|
19
19
|
end
|
20
20
|
end
|
data/lib/aspera/id_generator.rb
CHANGED
@@ -7,7 +7,7 @@ module Aspera
|
|
7
7
|
ID_SEPARATOR = '_'
|
8
8
|
WINDOWS_PROTECTED_CHAR = %r{[/:"<>\\*?]}.freeze
|
9
9
|
PROTECTED_CHAR_REPLACE = '_'
|
10
|
-
private_constant :ID_SEPARATOR
|
10
|
+
private_constant :ID_SEPARATOR, :PROTECTED_CHAR_REPLACE, :WINDOWS_PROTECTED_CHAR
|
11
11
|
class << self
|
12
12
|
def from_list(object_id)
|
13
13
|
if object_id.is_a?(Array)
|
@@ -16,10 +16,10 @@ module Aspera
|
|
16
16
|
end.join(ID_SEPARATOR)
|
17
17
|
end
|
18
18
|
raise 'id must be a String' unless object_id.is_a?(String)
|
19
|
-
return object_id
|
20
|
-
gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE)
|
21
|
-
gsub('.',PROTECTED_CHAR_REPLACE)
|
22
|
-
downcase
|
19
|
+
return object_id
|
20
|
+
.gsub(WINDOWS_PROTECTED_CHAR, PROTECTED_CHAR_REPLACE) # remove windows forbidden chars
|
21
|
+
.gsub('.', PROTECTED_CHAR_REPLACE) # keep dot for extension only (nicer)
|
22
|
+
.downcase
|
23
23
|
end
|
24
24
|
end
|
25
25
|
end
|
@@ -1,134 +1,67 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/hash_ext'
|
4
|
-
require '
|
4
|
+
require 'aspera/environment'
|
5
|
+
require 'symmetric_encryption/core'
|
6
|
+
require 'yaml'
|
5
7
|
|
6
8
|
module Aspera
|
7
9
|
module Keychain
|
8
|
-
class SimpleCipher
|
9
|
-
def initialize(key)
|
10
|
-
@key = Digest::SHA1.hexdigest(key+('*'*23))[0..23]
|
11
|
-
@cipher = OpenSSL::Cipher.new('DES-EDE3-CBC')
|
12
|
-
end
|
13
|
-
|
14
|
-
def encrypt(value)
|
15
|
-
@cipher.encrypt
|
16
|
-
@cipher.key = @key
|
17
|
-
s = @cipher.update(value) + @cipher.final
|
18
|
-
s.unpack1('H*')
|
19
|
-
end
|
20
|
-
|
21
|
-
def decrypt(value)
|
22
|
-
@cipher.decrypt
|
23
|
-
@cipher.key = @key
|
24
|
-
s = [value].pack('H*').unpack('C*').pack('c*')
|
25
|
-
@cipher.update(s) + @cipher.final
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
10
|
# Manage secrets in a simple Hash
|
30
11
|
class EncryptedHash
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
raise '
|
37
|
-
@all_secrets =
|
12
|
+
CIPHER_NAME = 'aes-256-cbc'
|
13
|
+
CONTENT_KEYS = %i[label username password url description].freeze
|
14
|
+
def initialize(path, current_password)
|
15
|
+
@path = path
|
16
|
+
self.password = current_password
|
17
|
+
raise 'path to vault file shall be String' unless @path.is_a?(String)
|
18
|
+
@all_secrets = File.exist?(@path) ? YAML.load_stream(@cipher.decrypt(File.read(@path))).first : {}
|
19
|
+
end
|
20
|
+
|
21
|
+
def password=(new_password)
|
22
|
+
# number of bits in second position
|
23
|
+
key_bytes = CIPHER_NAME.split('-')[1].to_i / Environment::BITS_PER_BYTE
|
24
|
+
# derive key from passphrase, add trailing zeros
|
25
|
+
key = "#{new_password}#{"\x0" * key_bytes}"[0..(key_bytes - 1)]
|
26
|
+
Log.log.debug{"key=[#{key}],#{key.length}"}
|
27
|
+
SymmetricEncryption.cipher = @cipher = SymmetricEncryption::Cipher.new(cipher_name: CIPHER_NAME, key: key, encoding: :none)
|
38
28
|
end
|
39
29
|
|
40
|
-
def
|
41
|
-
|
42
|
-
%i[url username].map{|s|options[s]}.join(SEPARATOR)
|
30
|
+
def save
|
31
|
+
File.write(@path, @cipher.encrypt(YAML.dump(@all_secrets)), encoding: 'BINARY')
|
43
32
|
end
|
44
33
|
|
45
34
|
def set(options)
|
46
35
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
47
|
-
unsupported = options.keys -
|
36
|
+
unsupported = options.keys - CONTENT_KEYS
|
37
|
+
options.each_value {|v| raise 'value must be String' unless v.is_a?(String)}
|
48
38
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
49
|
-
|
50
|
-
raise
|
51
|
-
|
52
|
-
|
53
|
-
secret = options[:secret]
|
54
|
-
raise 'options shall have secret' if secret.nil?
|
55
|
-
key = identifier(options)
|
56
|
-
raise "secret #{key} already exist, delete first" if @all_secrets.has_key?(key)
|
57
|
-
obj = {username: username, url: url, secret: SimpleCipher.new(key).encrypt(secret)}
|
58
|
-
obj[:description] = options[:description] if options.has_key?(:description)
|
59
|
-
@all_secrets[key] = obj.stringify_keys
|
60
|
-
nil
|
39
|
+
label = options.delete(:label)
|
40
|
+
raise "secret #{label} already exist, delete first" if @all_secrets.key?(label)
|
41
|
+
@all_secrets[label] = options.symbolize_keys
|
42
|
+
save
|
61
43
|
end
|
62
44
|
|
63
45
|
def list
|
64
46
|
result = []
|
65
|
-
|
66
|
-
|
67
|
-
normal =
|
68
|
-
|
69
|
-
when String
|
70
|
-
legacy_detected=true
|
71
|
-
{username: name, url: '', secret: value}
|
72
|
-
when Hash then value.symbolize_keys
|
73
|
-
else raise 'error secret must be String (legacy) or Hash (new)'
|
74
|
-
end
|
75
|
-
normal[:description] = '' unless normal.has_key?(:description)
|
76
|
-
extraneous_keys=normal.keys - ACCEPTED_KEYS
|
77
|
-
Log.log.error("wrongs keys in secret hash: #{extraneous_keys.map(&:to_s).join(',')}") unless extraneous_keys.empty?
|
47
|
+
@all_secrets.each do |label, values|
|
48
|
+
normal = values.symbolize_keys
|
49
|
+
normal[:label] = label
|
50
|
+
CONTENT_KEYS.each{|k|normal[k] = '' unless normal.key?(k)}
|
78
51
|
result.push(normal)
|
79
52
|
end
|
80
|
-
Log.log.warn('Legacy vault format detected in config file, please refer to documentation to convert to new format.') if legacy_detected
|
81
53
|
return result
|
82
54
|
end
|
83
55
|
|
84
|
-
def delete(
|
85
|
-
|
86
|
-
|
87
|
-
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
88
|
-
username = options[:username]
|
89
|
-
raise 'options shall have username' if username.nil?
|
90
|
-
url = options[:url]
|
91
|
-
key = nil
|
92
|
-
if !url.nil?
|
93
|
-
extk = identifier(options)
|
94
|
-
key = extk if @all_secrets.has_key?(extk)
|
95
|
-
end
|
96
|
-
# backward compatibility: TODO: remove in future ? (make url mandatory ?)
|
97
|
-
key = username if key.nil? && @all_secrets.has_key?(username)
|
98
|
-
raise 'no such secret' if key.nil?
|
99
|
-
@all_secrets.delete(key)
|
56
|
+
def delete(label:)
|
57
|
+
@all_secrets.delete(label)
|
58
|
+
save
|
100
59
|
end
|
101
60
|
|
102
|
-
def get(
|
103
|
-
raise
|
104
|
-
|
105
|
-
|
106
|
-
username = options[:username]
|
107
|
-
raise 'options shall have username' if username.nil?
|
108
|
-
url = options[:url]
|
109
|
-
info = nil
|
110
|
-
if !url.nil?
|
111
|
-
info = @all_secrets[identifier(options)]
|
112
|
-
end
|
113
|
-
# backward compatibility: TODO: remove in future ? (make url mandatory ?)
|
114
|
-
if info.nil?
|
115
|
-
info = @all_secrets[username]
|
116
|
-
end
|
117
|
-
result = options.clone
|
118
|
-
case info
|
119
|
-
when NilClass
|
120
|
-
raise "no such secret: [#{url}|#{username}] in #{@all_secrets.keys.join(',')}"
|
121
|
-
when String
|
122
|
-
result[:secret] = info
|
123
|
-
result[:description] = ''
|
124
|
-
when Hash
|
125
|
-
info=info.symbolize_keys
|
126
|
-
key = identifier(options)
|
127
|
-
plain = SimpleCipher.new(key).decrypt(info[:secret]) rescue info[:secret]
|
128
|
-
result[:secret] = plain
|
129
|
-
result[:description] = info[:description]
|
130
|
-
else raise "#{info.class} is not an expected type"
|
131
|
-
end
|
61
|
+
def get(label:, exception: true)
|
62
|
+
raise "Label not found: #{label}" unless @all_secrets.key?(label) || !exception
|
63
|
+
result = @all_secrets[label].clone
|
64
|
+
result[:label] = label if result.is_a?(Hash)
|
132
65
|
return result
|
133
66
|
end
|
134
67
|
end
|
@@ -1,91 +1,162 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
# https://github.com/fastlane-community/security
|
4
|
+
require 'aspera/cli/info'
|
4
5
|
|
5
6
|
# enhance the gem to support other keychains
|
6
|
-
module
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
7
|
+
module Aspera
|
8
|
+
module Keychain
|
9
|
+
module MacosSecurity
|
10
|
+
# keychain based on macOS keychain, using `security` cmmand line
|
11
|
+
class Keychain
|
12
|
+
DOMAINS = %i[user system common dynamic].freeze
|
13
|
+
LIST_OPTIONS = {
|
14
|
+
domain: :c
|
15
|
+
}
|
16
|
+
ADD_PASS_OPTIONS = {
|
17
|
+
account: :a,
|
18
|
+
creator: :c,
|
19
|
+
type: :C,
|
20
|
+
domain: :d,
|
21
|
+
kind: :D,
|
22
|
+
value: :G,
|
23
|
+
comment: :j,
|
24
|
+
label: :l,
|
25
|
+
path: :p,
|
26
|
+
port: :P,
|
27
|
+
protocol: :r,
|
28
|
+
server: :s,
|
29
|
+
service: :s,
|
30
|
+
auth: :t,
|
31
|
+
password: :w,
|
32
|
+
getpass: :g
|
33
|
+
}.freeze
|
34
|
+
class << self
|
35
|
+
def execute(command, options=nil, supported=nil, lastopt=nil)
|
36
|
+
url = options&.delete(:url)
|
37
|
+
if !url.nil?
|
38
|
+
uri = URI.parse(url)
|
39
|
+
raise 'only https' unless uri.scheme.eql?('https')
|
40
|
+
options[:protocol] = 'htps'
|
41
|
+
raise 'host required in URL' if uri.host.nil?
|
42
|
+
options[:server] = uri.host
|
43
|
+
options[:path] = uri.path unless ['', '/'].include?(uri.path)
|
44
|
+
options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
|
45
|
+
end
|
46
|
+
cmd = ['security', command]
|
47
|
+
options&.each do |k, v|
|
48
|
+
raise "unknown option: #{k}" unless supported.key?(k)
|
49
|
+
next if v.nil?
|
50
|
+
cmd.push("-#{supported[k]}")
|
51
|
+
cmd.push(v.shellescape) unless v.empty?
|
52
|
+
end
|
53
|
+
cmd.push(lastopt) unless lastopt.nil?
|
54
|
+
Log.log.debug{"executing>>#{cmd.join(' ')}"}
|
55
|
+
result = %x(#{cmd.join(' ')} 2>&1)
|
56
|
+
Log.log.debug{"result>>[#{result}]"}
|
57
|
+
return result
|
58
|
+
end
|
14
59
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
options
|
28
|
-
|
29
|
-
|
60
|
+
def keychains(output)
|
61
|
+
output.split("\n").collect { |line| new(line.strip.gsub(/^"|"$/, '')) }
|
62
|
+
end
|
63
|
+
|
64
|
+
def default
|
65
|
+
keychains(execute('default-keychain')).first
|
66
|
+
end
|
67
|
+
|
68
|
+
def login
|
69
|
+
keychains(execute('login-keychain')).first
|
70
|
+
end
|
71
|
+
|
72
|
+
def list(options={})
|
73
|
+
raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
|
74
|
+
keychains(execute('list-keychains', options, LIST_OPTIONS))
|
75
|
+
end
|
76
|
+
|
77
|
+
def by_name(name)
|
78
|
+
list.find{|kc|kc.path.end_with?("/#{name}.keychain-db")}
|
79
|
+
end
|
80
|
+
end
|
81
|
+
attr_reader :path
|
82
|
+
|
83
|
+
def initialize(path)
|
84
|
+
@path = path
|
85
|
+
end
|
86
|
+
|
87
|
+
def decode_hex_blob(string)
|
88
|
+
[string].pack('H*').force_encoding('UTF-8')
|
89
|
+
end
|
90
|
+
|
91
|
+
def password(operation, passtype, options)
|
92
|
+
raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
|
93
|
+
raise "wrong passtype: #{passtype}" unless %i[generic internet].include?(passtype)
|
94
|
+
raise 'options shall be Hash' unless options.is_a?(Hash)
|
95
|
+
missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
|
96
|
+
raise "missing options: #{missing}" unless missing.empty?
|
97
|
+
options[:getpass] = '' if operation.eql?(:find)
|
98
|
+
output = self.class.execute("#{operation}-#{passtype}-password", options, ADD_PASS_OPTIONS, @path)
|
99
|
+
raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
|
100
|
+
return nil unless operation.eql?(:find)
|
101
|
+
attributes = {}
|
102
|
+
output.split("\n").each do |line|
|
103
|
+
case line
|
104
|
+
when /^keychain: "(.+)"/
|
105
|
+
# ignore
|
106
|
+
when /0x00000007 .+="(.+)"/
|
107
|
+
attributes['label'] = Regexp.last_match(1)
|
108
|
+
when /"(\w{4})".+="(.+)"/
|
109
|
+
attributes[Regexp.last_match(1)] = Regexp.last_match(2)
|
110
|
+
when /"(\w{4})"<blob>=0x([[:xdigit:]]+)/
|
111
|
+
attributes[Regexp.last_match(1)] = decode_hex_blob(Regexp.last_match(2))
|
112
|
+
when /^password: "(.+)"/
|
113
|
+
attributes['password'] = Regexp.last_match(1)
|
114
|
+
when /^password: 0x([[:xdigit:]]+)/
|
115
|
+
attributes['password'] = decode_hex_blob(Regexp.last_match(1))
|
116
|
+
end
|
117
|
+
end
|
118
|
+
return attributes
|
30
119
|
end
|
31
|
-
flags = [orig_flags_for_options(options)]
|
32
|
-
flags.push(keychain.filename) unless keychain.nil?
|
33
|
-
flags.join(' ')
|
34
120
|
end
|
35
121
|
end
|
36
|
-
end
|
37
|
-
end
|
38
122
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
class MacosSecurity
|
43
|
-
def initialize(name=nil)
|
44
|
-
@keychain = name.nil? ? Security::Keychain.default_keychain : Security::Keychain.by_name(name)
|
123
|
+
class MacosSystem
|
124
|
+
def initialize(name=nil, _password=nil)
|
125
|
+
@keychain = name.nil? ? MacosSecurity::Keychain.default_keychain : MacosSecurity::Keychain.by_name(name)
|
45
126
|
raise "no such keychain #{name}" if @keychain.nil?
|
46
127
|
end
|
47
128
|
|
48
129
|
def set(options)
|
49
130
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
50
|
-
unsupported = options.keys - %i[username url
|
131
|
+
unsupported = options.keys - %i[label username password url description]
|
51
132
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
raise 'options shall have url' if url.nil?
|
56
|
-
secret = options[:secret]
|
57
|
-
raise 'options shall have secret' if secret.nil?
|
58
|
-
raise 'set not implemented'
|
133
|
+
@keychain.password(
|
134
|
+
:add, :generic, service: options[:label],
|
135
|
+
account: options[:username] || 'none', password: options[:password], comment: options[:description])
|
59
136
|
end
|
60
137
|
|
61
138
|
def get(options)
|
62
139
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
63
|
-
unsupported = options.keys - %i[
|
140
|
+
unsupported = options.keys - %i[label]
|
64
141
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
65
|
-
|
66
|
-
raise 'options shall have username' if username.nil?
|
67
|
-
url = options[:url]
|
68
|
-
raise 'options shall have url' if url.nil?
|
69
|
-
info = Security::InternetPassword.find(keychain: @keychain, url: url, account: username)
|
142
|
+
info = @keychain.password(:find, :generic, label: options[:label])
|
70
143
|
raise 'not found' if info.nil?
|
71
144
|
result = options.clone
|
72
|
-
result[:secret] = info
|
73
|
-
result[:description] = info
|
145
|
+
result[:secret] = info['password']
|
146
|
+
result[:description] = info['icmt']
|
74
147
|
return result
|
75
148
|
end
|
76
149
|
|
77
150
|
def list
|
78
|
-
|
151
|
+
# the only way to list is `dump-keychain` which triggers security alert
|
152
|
+
raise 'list not implemented, use macos keychain app'
|
79
153
|
end
|
80
154
|
|
81
155
|
def delete(options)
|
82
156
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
83
|
-
unsupported = options.keys - %i[
|
157
|
+
unsupported = options.keys - %i[label]
|
84
158
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
85
|
-
|
86
|
-
raise 'options shall have username' if username.nil?
|
87
|
-
url = options[:url]
|
88
|
-
raise "delete not implemented #{url}"
|
159
|
+
raise 'delete not implemented, use macos keychain app'
|
89
160
|
end
|
90
161
|
end
|
91
162
|
end
|
data/lib/aspera/log.rb
CHANGED
@@ -14,25 +14,25 @@ module Aspera
|
|
14
14
|
# class methods
|
15
15
|
class << self
|
16
16
|
# levels are :debug,:info,:warn,:error,fatal,:unknown
|
17
|
-
def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
|
17
|
+
def levels; Logger::Severity.constants.sort{|a, b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym}; end
|
18
18
|
|
19
19
|
# where logs are sent to
|
20
|
-
def logtypes; %i[stderr stdout syslog];end
|
20
|
+
def logtypes; %i[stderr stdout syslog]; end
|
21
21
|
|
22
22
|
# get the logger object of singleton
|
23
|
-
def log; instance.logger;end
|
23
|
+
def log; instance.logger; end
|
24
24
|
|
25
25
|
# dump object in debug mode
|
26
26
|
# @param name string or symbol
|
27
27
|
# @param format either pp or json format
|
28
|
-
def dump(name,object,format=:json)
|
28
|
+
def dump(name, object, format=:json)
|
29
29
|
log.debug do
|
30
30
|
result =
|
31
31
|
case format
|
32
32
|
when :json
|
33
|
-
JSON.pretty_generate(object) rescue PP.pp(object
|
33
|
+
JSON.pretty_generate(object) rescue PP.pp(object, +'')
|
34
34
|
when :ruby
|
35
|
-
PP.pp(object
|
35
|
+
PP.pp(object, +'')
|
36
36
|
else
|
37
37
|
raise 'wrong parameter, expect pp or json'
|
38
38
|
end
|
@@ -70,7 +70,7 @@ module Aspera
|
|
70
70
|
# change underlying logger, but keep log level
|
71
71
|
def logger_type=(new_logtype)
|
72
72
|
current_severity_integer = @logger.level unless @logger.nil?
|
73
|
-
current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.
|
73
|
+
current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
|
74
74
|
current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
|
75
75
|
case new_logtype
|
76
76
|
when :stderr
|