aspera-cli 4.20.0 → 4.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +29 -3
- data/CONTRIBUTING.md +2 -0
- data/README.md +571 -375
- data/bin/asession +2 -2
- data/examples/get_proto_file.rb +1 -1
- data/lib/aspera/agent/alpha.rb +10 -16
- data/lib/aspera/agent/connect.rb +20 -2
- data/lib/aspera/agent/direct.rb +21 -30
- data/lib/aspera/agent/node.rb +1 -11
- data/lib/aspera/agent/{trsdk.rb → transferd.rb} +13 -34
- data/lib/aspera/api/aoc.rb +13 -8
- data/lib/aspera/api/node.rb +45 -28
- data/lib/aspera/ascp/installation.rb +87 -48
- data/lib/aspera/ascp/management.rb +27 -6
- data/lib/aspera/cli/formatter.rb +148 -154
- data/lib/aspera/cli/info.rb +1 -1
- data/lib/aspera/cli/main.rb +12 -0
- data/lib/aspera/cli/manager.rb +2 -2
- data/lib/aspera/cli/plugin.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +28 -18
- data/lib/aspera/cli/plugins/config.rb +106 -54
- data/lib/aspera/cli/plugins/cos.rb +1 -0
- data/lib/aspera/cli/plugins/faspex.rb +4 -2
- data/lib/aspera/cli/plugins/faspex5.rb +21 -9
- data/lib/aspera/cli/plugins/node.rb +45 -38
- data/lib/aspera/cli/transfer_progress.rb +2 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +1 -1
- data/lib/aspera/environment.rb +48 -14
- data/lib/aspera/node_simulator.rb +230 -112
- data/lib/aspera/oauth/base.rb +34 -47
- data/lib/aspera/oauth/factory.rb +41 -2
- data/lib/aspera/oauth/jwt.rb +4 -1
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/persistency_folder.rb +20 -2
- data/lib/aspera/preview/generator.rb +1 -1
- data/lib/aspera/preview/utils.rb +8 -3
- data/lib/aspera/products/alpha.rb +30 -0
- data/lib/aspera/products/connect.rb +48 -0
- data/lib/aspera/products/other.rb +82 -0
- data/lib/aspera/products/transferd.rb +54 -0
- data/lib/aspera/rest.rb +18 -13
- data/lib/aspera/secret_hider.rb +2 -2
- data/lib/aspera/ssh.rb +31 -24
- data/lib/aspera/transfer/parameters.rb +2 -1
- data/lib/aspera/transfer/spec.yaml +22 -20
- data/lib/aspera/transfer/sync.rb +1 -5
- data/lib/aspera/transfer/uri.rb +2 -2
- data/lib/transferd_pb.rb +86 -0
- data/lib/transferd_services_pb.rb +84 -0
- data.tar.gz.sig +0 -0
- metadata +38 -21
- metadata.gz.sig +0 -0
- data/lib/aspera/ascp/products.rb +0 -168
- data/lib/transfer_pb.rb +0 -84
- data/lib/transfer_services_pb.rb +0 -82
data/lib/aspera/oauth/factory.rb
CHANGED
@@ -13,6 +13,7 @@ module Aspera
|
|
13
13
|
PERSIST_CATEGORY_TOKEN = 'token'
|
14
14
|
# prefix for bearer token when in header
|
15
15
|
BEARER_PREFIX = 'Bearer '
|
16
|
+
TOKEN_FIELD = 'access_token'
|
16
17
|
|
17
18
|
private_constant :PERSIST_CATEGORY_TOKEN, :BEARER_PREFIX
|
18
19
|
|
@@ -87,7 +88,45 @@ module Aspera
|
|
87
88
|
|
88
89
|
# delete all existing tokens
|
89
90
|
def flush_tokens
|
90
|
-
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN
|
91
|
+
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN)
|
92
|
+
end
|
93
|
+
|
94
|
+
def persisted_tokens
|
95
|
+
data = persist_mgr.current_items(PERSIST_CATEGORY_TOKEN)
|
96
|
+
data.each.map do |k, v|
|
97
|
+
info = {id: k}
|
98
|
+
info.merge!(JSON.parse(v)) rescue nil
|
99
|
+
d = decode_token(info.delete(TOKEN_FIELD))
|
100
|
+
info.merge(d) if d
|
101
|
+
info
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
# get token information from cache
|
106
|
+
# @param id [String] identifier of token
|
107
|
+
# @return [Hash] token internal information , including Date object for `expiration_date`
|
108
|
+
def get_token_info(id)
|
109
|
+
token_raw_string = persist_mgr.get(id)
|
110
|
+
return nil if token_raw_string.nil?
|
111
|
+
token_data = JSON.parse(token_raw_string)
|
112
|
+
Aspera.assert_type(token_data, Hash)
|
113
|
+
decoded_token = decode_token(token_data[TOKEN_FIELD])
|
114
|
+
info = { data: token_data }
|
115
|
+
Log.log.debug{Log.dump('decoded_token', decoded_token)}
|
116
|
+
if decoded_token.is_a?(Hash)
|
117
|
+
info[:decoded] = decoded_token
|
118
|
+
# TODO: move date decoding to token decoder ?
|
119
|
+
expiration_date =
|
120
|
+
if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
|
121
|
+
elsif decoded_token['exp'].is_a?(Integer) then Time.at(decoded_token['exp'])
|
122
|
+
end
|
123
|
+
unless expiration_date.nil?
|
124
|
+
info[:expiration] = expiration_date
|
125
|
+
info[:ttl_sec] = expiration_date - Time.now
|
126
|
+
info[:expired] = info[:ttl_sec] < @parameters[:token_expiration_guard_sec]
|
127
|
+
end
|
128
|
+
end
|
129
|
+
return info
|
91
130
|
end
|
92
131
|
|
93
132
|
# register a bearer token decoder, mainly to inspect expiry date
|
@@ -125,6 +164,6 @@ module Aspera
|
|
125
164
|
end
|
126
165
|
end
|
127
166
|
# JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
|
128
|
-
Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not
|
167
|
+
Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert(parts.length.eql?(3)){'not JWS token'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon, Layout/LineLength
|
129
168
|
end
|
130
169
|
end
|
data/lib/aspera/oauth/jwt.rb
CHANGED
@@ -21,12 +21,15 @@ module Aspera
|
|
21
21
|
private_key_obj:,
|
22
22
|
payload:,
|
23
23
|
headers: {},
|
24
|
+
cache_ids: [],
|
24
25
|
**base_params
|
25
26
|
)
|
26
27
|
Aspera.assert_type(private_key_obj, OpenSSL::PKey::RSA){'private_key_obj'}
|
27
28
|
Aspera.assert_type(payload, Hash){'payload'}
|
28
29
|
Aspera.assert_type(headers, Hash){'headers'}
|
29
|
-
|
30
|
+
Aspera.assert_type(cache_ids, Array){'cache ids'}
|
31
|
+
new_cache_ids = cache_ids.clone.push(payload[:sub])
|
32
|
+
super(**base_params, cache_ids: new_cache_ids)
|
30
33
|
@private_key_obj = private_key_obj
|
31
34
|
@additional_payload = payload
|
32
35
|
@headers = headers
|
@@ -7,7 +7,7 @@ require 'aspera/assert'
|
|
7
7
|
module Aspera
|
8
8
|
# Persist data on file system
|
9
9
|
class PersistencyActionOnce
|
10
|
-
DELETE_DEFAULT = lambda
|
10
|
+
DELETE_DEFAULT = lambda(&:empty?)
|
11
11
|
PARSE_DEFAULT = lambda {|t| JSON.parse(t)}
|
12
12
|
FORMAT_DEFAULT = lambda {|h| JSON.generate(h)}
|
13
13
|
MERGE_DEFAULT = lambda {|current, file| current.concat(file).uniq rescue current}
|
@@ -18,7 +18,8 @@ module Aspera
|
|
18
18
|
Log.log.debug{"persistency folder: #{@folder}"}
|
19
19
|
end
|
20
20
|
|
21
|
-
#
|
21
|
+
# Get value of persisted item
|
22
|
+
# @return [String,nil] Value of persisted id
|
22
23
|
def get(object_id)
|
23
24
|
Log.log.debug{"persistency get: #{object_id}"}
|
24
25
|
if @cache.key?(object_id)
|
@@ -34,6 +35,10 @@ module Aspera
|
|
34
35
|
return @cache[object_id]
|
35
36
|
end
|
36
37
|
|
38
|
+
# Set value of persisted item
|
39
|
+
# @param object_id [String] Identifier of persisted item
|
40
|
+
# @param value [String] Value of persisted item
|
41
|
+
# @return [nil]
|
37
42
|
def put(object_id, value)
|
38
43
|
Aspera.assert_type(value, String)
|
39
44
|
persist_filepath = id_to_filepath(object_id)
|
@@ -42,8 +47,11 @@ module Aspera
|
|
42
47
|
File.write(persist_filepath, value)
|
43
48
|
Environment.restrict_file_access(persist_filepath)
|
44
49
|
@cache[object_id] = value
|
50
|
+
nil
|
45
51
|
end
|
46
52
|
|
53
|
+
# Delete persisted item
|
54
|
+
# @param object_id [String] Identifier of persisted item
|
47
55
|
def delete(object_id)
|
48
56
|
persist_filepath = id_to_filepath(object_id)
|
49
57
|
Log.log.debug{"persistency deleting: #{persist_filepath}"}
|
@@ -51,8 +59,9 @@ module Aspera
|
|
51
59
|
@cache.delete(object_id)
|
52
60
|
end
|
53
61
|
|
62
|
+
# Delete persisted items
|
54
63
|
def garbage_collect(persist_category, max_age_seconds=nil)
|
55
|
-
garbage_files =
|
64
|
+
garbage_files = current_files(persist_category)
|
56
65
|
if !max_age_seconds.nil?
|
57
66
|
current_time = Time.now
|
58
67
|
garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
|
@@ -61,9 +70,18 @@ module Aspera
|
|
61
70
|
File.delete(filepath)
|
62
71
|
Log.log.debug{"persistency deleted expired: #{filepath}"}
|
63
72
|
end
|
73
|
+
@cache.clear
|
64
74
|
return garbage_files
|
65
75
|
end
|
66
76
|
|
77
|
+
def current_files(persist_category)
|
78
|
+
Dir[File.join(@folder, persist_category + '*' + FILE_SUFFIX)]
|
79
|
+
end
|
80
|
+
|
81
|
+
def current_items(persist_category)
|
82
|
+
current_files(persist_category).each_with_object({}) {|i, h| h[File.basename(i, FILE_SUFFIX)] = File.read(i)}
|
83
|
+
end
|
84
|
+
|
67
85
|
private
|
68
86
|
|
69
87
|
# @param object_id String or Array
|
@@ -66,7 +66,7 @@ module Aspera
|
|
66
66
|
result_size = File.size(@destination_file_path)
|
67
67
|
Log.log.warn{"preview size exceeds maximum allowed #{result_size} > #{@options.max_size}"} if result_size > @options.max_size
|
68
68
|
rescue StandardError => e
|
69
|
-
Log.log.error{"Ignoring: #{e.message}"}
|
69
|
+
Log.log.error{"Ignoring: #{e.class} #{e.message}"}
|
70
70
|
Log.log.debug(e.backtrace.join("\n").red)
|
71
71
|
FileUtils.cp(File.expand_path(@preview_format_sym.eql?(:mp4) ? 'video_error.png' : 'image_error.png', File.dirname(__FILE__)), @destination_file_path)
|
72
72
|
ensure
|
data/lib/aspera/preview/utils.rb
CHANGED
@@ -42,10 +42,15 @@ module Aspera
|
|
42
42
|
|
43
43
|
# execute external command
|
44
44
|
# one could use "system", but we would need to redirect stdout/err
|
45
|
-
# @return
|
45
|
+
# @return nil
|
46
46
|
def external_command(command_sym, command_args)
|
47
47
|
Aspera.assert_values(command_sym, EXTERNAL_TOOLS){'command'}
|
48
|
-
|
48
|
+
Environment.secure_execute(exec: command_sym.to_s, args: command_args.map(&:to_s))
|
49
|
+
end
|
50
|
+
|
51
|
+
def external_capture(command_sym, command_args)
|
52
|
+
Aspera.assert_values(command_sym, EXTERNAL_TOOLS){'command'}
|
53
|
+
return Environment.secure_capture(exec: command_sym.to_s, args: command_args.map(&:to_s))
|
49
54
|
end
|
50
55
|
|
51
56
|
def ffmpeg(a)
|
@@ -63,7 +68,7 @@ module Aspera
|
|
63
68
|
|
64
69
|
# @return Float in seconds
|
65
70
|
def video_get_duration(input_file)
|
66
|
-
return
|
71
|
+
return external_capture(:ffprobe, [
|
67
72
|
'-loglevel', 'error',
|
68
73
|
'-show_entries', 'format=duration',
|
69
74
|
'-print_format', 'default=noprint_wrappers=1:nokey=1', # cspell:disable-line
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/environment'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
module Products
|
7
|
+
# Aspera Desktop Alpha Client
|
8
|
+
class Alpha
|
9
|
+
APP_NAME = 'IBM Aspera for Desktop'
|
10
|
+
APP_IDENTIFIER = 'com.ibm.software.aspera.desktop'
|
11
|
+
class << self
|
12
|
+
# standard folder locations
|
13
|
+
def locations
|
14
|
+
case Aspera::Environment.os
|
15
|
+
when Aspera::Environment::OS_MACOS then [{
|
16
|
+
app_root: File.join('', 'Applications', 'IBM Aspera.app'),
|
17
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER),
|
18
|
+
sub_bin: File.join('Contents', 'Resources', 'transferd', 'bin')
|
19
|
+
}]
|
20
|
+
else []
|
21
|
+
end.map { |i| i.merge({ expected: APP_NAME }) }
|
22
|
+
end
|
23
|
+
|
24
|
+
def log_file
|
25
|
+
File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER, 'ibm-aspera-desktop.log')
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/environment'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
module Products
|
7
|
+
class Connect
|
8
|
+
APP_NAME = 'IBM Aspera Connect'
|
9
|
+
class << self
|
10
|
+
# standard folder locations
|
11
|
+
def locations
|
12
|
+
case Aspera::Environment.os
|
13
|
+
when Aspera::Environment::OS_WINDOWS then [{
|
14
|
+
app_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Programs', 'Aspera', 'Aspera Connect'),
|
15
|
+
log_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Aspera', 'Aspera Connect', 'var', 'log'),
|
16
|
+
run_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Aspera', 'Aspera Connect')
|
17
|
+
}]
|
18
|
+
when Aspera::Environment::OS_MACOS then [{
|
19
|
+
app_root: File.join(Dir.home, 'Applications', 'Aspera Connect.app'),
|
20
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
|
21
|
+
run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
|
22
|
+
sub_bin: File.join('Contents', 'Resources')
|
23
|
+
}, {
|
24
|
+
app_root: File.join('', 'Applications', 'Aspera Connect.app'),
|
25
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
|
26
|
+
run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
|
27
|
+
sub_bin: File.join('Contents', 'Resources')
|
28
|
+
}, {
|
29
|
+
app_root: File.join(Dir.home, 'Applications', 'IBM Aspera Connect.app'),
|
30
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
|
31
|
+
run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
|
32
|
+
sub_bin: File.join('Contents', 'Resources')
|
33
|
+
}, {
|
34
|
+
app_root: File.join('', 'Applications', 'IBM Aspera Connect.app'),
|
35
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
|
36
|
+
run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
|
37
|
+
sub_bin: File.join('Contents', 'Resources')
|
38
|
+
}]
|
39
|
+
else [{ # other: Linux and Unix family
|
40
|
+
app_root: File.join(Dir.home, '.aspera', 'connect'),
|
41
|
+
run_root: File.join(Dir.home, '.aspera', 'connect')
|
42
|
+
}]
|
43
|
+
end.map { |i| i.merge({ expected: APP_NAME }) }
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# cspell:ignore LOCALAPPDATA
|
4
|
+
require 'aspera/environment'
|
5
|
+
|
6
|
+
module Aspera
|
7
|
+
# Location of Aspera products, for which an Agent is not proposed
|
8
|
+
module Products
|
9
|
+
# other Aspera products with ascp
|
10
|
+
class Other
|
11
|
+
CLI_V3 = 'Aspera CLI (deprecated)'
|
12
|
+
DRIVE = 'Aspera Drive (deprecated)'
|
13
|
+
HSTS = 'IBM Aspera High-Speed Transfer Server'
|
14
|
+
|
15
|
+
private_constant :CLI_V3, :DRIVE, :HSTS
|
16
|
+
# product information manifest: XML (part of aspera product)
|
17
|
+
INFO_META_FILE = 'product-info.mf'
|
18
|
+
|
19
|
+
# :expected M app name is taken from the manifest if present, else defaults to this value
|
20
|
+
# :app_root M main folder for the application
|
21
|
+
# :log_root O location of log files (Linux uses syslog)
|
22
|
+
# :run_root O only for Connect Client, location of http port file
|
23
|
+
# :sub_bin O subfolder with executables, default : bin
|
24
|
+
LOCATION_ON_THIS_OS = case Aspera::Environment.os
|
25
|
+
when Aspera::Environment::OS_WINDOWS then [{
|
26
|
+
expected: CLI_V3,
|
27
|
+
app_root: File.join('C:', 'Program Files', 'Aspera', 'cli'),
|
28
|
+
log_root: File.join('C:', 'Program Files', 'Aspera', 'cli', 'var', 'log')
|
29
|
+
}, {
|
30
|
+
expected: HSTS,
|
31
|
+
app_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server'),
|
32
|
+
log_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server', 'var', 'log')
|
33
|
+
}]
|
34
|
+
when Aspera::Environment::OS_MACOS then [{
|
35
|
+
expected: CLI_V3,
|
36
|
+
app_root: File.join(Dir.home, 'Applications', 'Aspera CLI'),
|
37
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera')
|
38
|
+
}, {
|
39
|
+
expected: HSTS,
|
40
|
+
app_root: File.join('', 'Library', 'Aspera'),
|
41
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera')
|
42
|
+
}, {
|
43
|
+
expected: DRIVE,
|
44
|
+
app_root: File.join('', 'Applications', 'Aspera Drive.app'),
|
45
|
+
log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Drive'),
|
46
|
+
sub_bin: File.join('Contents', 'Resources')
|
47
|
+
}]
|
48
|
+
else [{ # other: Linux and Unix family
|
49
|
+
expected: CLI_V3,
|
50
|
+
app_root: File.join(Dir.home, '.aspera', 'cli')
|
51
|
+
}, {
|
52
|
+
expected: HSTS,
|
53
|
+
app_root: File.join('', 'opt', 'aspera')
|
54
|
+
}]
|
55
|
+
end
|
56
|
+
class << self
|
57
|
+
def find(scan_locations)
|
58
|
+
scan_locations.select do |item|
|
59
|
+
# skip if not main folder
|
60
|
+
Log.log.trace1{"Checking #{item[:app_root]}"}
|
61
|
+
next false unless Dir.exist?(item[:app_root])
|
62
|
+
Log.log.debug{"Found #{item[:expected]}"}
|
63
|
+
sub_bin = item[:sub_bin] || 'bin'
|
64
|
+
item[:ascp_path] = File.join(item[:app_root], sub_bin, Environment.exe_file('ascp'))
|
65
|
+
# skip if no ascp
|
66
|
+
next false unless File.exist?(item[:ascp_path])
|
67
|
+
# read info from product info file if present
|
68
|
+
product_info_file = "#{item[:app_root]}/#{INFO_META_FILE}"
|
69
|
+
if File.exist?(product_info_file)
|
70
|
+
res_s = XmlSimple.xml_in(File.read(product_info_file), {'ForceArray' => false})
|
71
|
+
item[:name] = res_s['name']
|
72
|
+
item[:version] = res_s['version']
|
73
|
+
else
|
74
|
+
item[:name] = item[:expected]
|
75
|
+
end
|
76
|
+
true # select this version
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Aspera
|
4
|
+
module Products
|
5
|
+
class Transferd
|
6
|
+
APP_NAME = 'IBM Aspera Transfer Daemon'
|
7
|
+
class << self
|
8
|
+
# standard folder locations
|
9
|
+
def locations
|
10
|
+
[{
|
11
|
+
app_root: sdk_directory,
|
12
|
+
sub_bin: ''
|
13
|
+
}].map { |i| i.merge({ expected: APP_NAME }) }
|
14
|
+
end
|
15
|
+
|
16
|
+
# location of SDK files
|
17
|
+
def sdk_directory=(v)
|
18
|
+
Log.log.debug{"sdk_directory=#{v}"}
|
19
|
+
@sdk_dir = v
|
20
|
+
sdk_directory
|
21
|
+
end
|
22
|
+
|
23
|
+
# @return the path to folder where SDK is installed
|
24
|
+
def sdk_directory
|
25
|
+
Aspera.assert(!@sdk_dir.nil?){'SDK path was not initialized'}
|
26
|
+
FileUtils.mkdir_p(@sdk_dir)
|
27
|
+
@sdk_dir
|
28
|
+
end
|
29
|
+
|
30
|
+
def transferd_path
|
31
|
+
return File.join(sdk_directory, Environment.exe_file('asperatransferd')) # cspell:disable-line
|
32
|
+
end
|
33
|
+
|
34
|
+
# Well, the port number is only in log file
|
35
|
+
def daemon_port_from_log(log_file)
|
36
|
+
result = nil
|
37
|
+
# if port is zero, a dynamic port was created, get it
|
38
|
+
File.open(log_file, 'r') do |file|
|
39
|
+
file.each_line do |line|
|
40
|
+
# Well, it's tricky to depend on log
|
41
|
+
if (m = line.match(/Info: API Server: Listening on ([^:]+):(\d+) /))
|
42
|
+
result = m[2].to_i
|
43
|
+
# no "break" , need to read last matching log line
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
raise 'Port not found in daemon logs' if result.nil?
|
48
|
+
Log.log.debug{"Got port #{result} from log"}
|
49
|
+
return result
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
data/lib/aspera/rest.rb
CHANGED
@@ -10,7 +10,6 @@ require 'net/http'
|
|
10
10
|
require 'net/https'
|
11
11
|
require 'json'
|
12
12
|
require 'base64'
|
13
|
-
require 'cgi'
|
14
13
|
require 'singleton'
|
15
14
|
require 'securerandom'
|
16
15
|
|
@@ -22,11 +21,13 @@ class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModul
|
|
22
21
|
end
|
23
22
|
|
24
23
|
module Aspera
|
25
|
-
# Global settings
|
24
|
+
# Global settings for Rest object
|
25
|
+
# For example to remove certificate verification globally:
|
26
|
+
# `RestParameters.instance.session_cb = lambda{|http|http.verify_mode=OpenSSL::SSL::VERIFY_NONE}`
|
26
27
|
# @param user_agent [String] HTTP request header: 'User-Agent'
|
27
28
|
# @param download_partial_suffix [String] suffix for partial download
|
28
29
|
# @param session_cb [lambda] lambda called on new HTTP session. Takes the Net::HTTP as arg. Used to change parameters on creation.
|
29
|
-
# @param progress_bar [Object] progress bar object
|
30
|
+
# @param progress_bar [Object] progress bar object called for file transfer
|
30
31
|
class RestParameters
|
31
32
|
include Singleton
|
32
33
|
|
@@ -200,6 +201,9 @@ module Aspera
|
|
200
201
|
}
|
201
202
|
end
|
202
203
|
|
204
|
+
# Create a REST object for API calls
|
205
|
+
# HTTP sessions parameters can be modified using global parameters in RestParameters
|
206
|
+
# For example, TLS verification can be skipped.
|
203
207
|
# @param base_url [String] base URL of REST API
|
204
208
|
# @param auth [Hash] authentication parameters:
|
205
209
|
# :type (:none, :basic, :url, :oauth2)
|
@@ -207,14 +211,15 @@ module Aspera
|
|
207
211
|
# :password [:basic]
|
208
212
|
# :url_query [:url] a hash
|
209
213
|
# :* [:oauth2] see OAuth::Factory class
|
210
|
-
# @param not_auth_codes [Array]
|
211
|
-
# @param redirect_max
|
214
|
+
# @param not_auth_codes [Array] codes that trigger a refresh/regeneration of bearer token
|
215
|
+
# @param redirect_max [Integer] max redirection allowed
|
216
|
+
# @param headers [Hash] default headers to include in all calls
|
212
217
|
def initialize(
|
213
218
|
base_url:,
|
214
|
-
auth:
|
215
|
-
not_auth_codes:
|
219
|
+
auth: {type: :none},
|
220
|
+
not_auth_codes: ['401'],
|
216
221
|
redirect_max: 0,
|
217
|
-
headers:
|
222
|
+
headers: {}
|
218
223
|
)
|
219
224
|
Aspera.assert_type(base_url, String)
|
220
225
|
# base url with no trailing slashes (note: string may be frozen)
|
@@ -224,20 +229,20 @@ module Aspera
|
|
224
229
|
@base_url = @base_url.gsub(/:80$/, '') if @base_url.start_with?('http://')
|
225
230
|
Log.log.debug{"Rest.new(#{@base_url})"}
|
226
231
|
# default is no auth
|
227
|
-
@auth_params = auth
|
232
|
+
@auth_params = auth
|
228
233
|
Aspera.assert_type(@auth_params, Hash)
|
229
234
|
Aspera.assert(@auth_params.key?(:type)){'no auth type defined'}
|
230
|
-
@not_auth_codes = not_auth_codes
|
235
|
+
@not_auth_codes = not_auth_codes
|
231
236
|
Aspera.assert_type(@not_auth_codes, Array)
|
232
237
|
# persistent session
|
233
238
|
@http_session = nil
|
234
|
-
# OAuth object (created on demand)
|
235
|
-
@oauth = nil
|
236
239
|
@redirect_max = redirect_max
|
237
240
|
Aspera.assert_type(@redirect_max, Integer)
|
238
|
-
@headers = headers
|
241
|
+
@headers = headers
|
239
242
|
Aspera.assert_type(@headers, Hash)
|
240
243
|
@headers['User-Agent'] ||= RestParameters.instance.user_agent
|
244
|
+
# OAuth object (created on demand)
|
245
|
+
@oauth = nil
|
241
246
|
end
|
242
247
|
|
243
248
|
# @return the OAuth object (create, or cached if already created)
|
data/lib/aspera/secret_hider.rb
CHANGED
@@ -20,6 +20,8 @@ module Aspera
|
|
20
20
|
KEY_FALSE_POSITIVES = [/^access_key$/, /^fallback_private_key$/].freeze
|
21
21
|
# regex that define named captures :begin and :end
|
22
22
|
REGEX_LOG_REPLACES = [
|
23
|
+
# private key values (place first)
|
24
|
+
/(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)\n*/,
|
23
25
|
# CLI manager get/set options
|
24
26
|
/(?<begin>[sg]et (?:#{KEY_SECRETS.join('|')})=).*(?<end>)/,
|
25
27
|
# env var ascp exec
|
@@ -28,8 +30,6 @@ module Aspera
|
|
28
30
|
/(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>?(?:=>|:) *")[^"]+(?<end>")/,
|
29
31
|
# logged data
|
30
32
|
/(?<begin>(?:#{ALL_SECRETS2.join('|')})[ =:]+).*(?<end>$)/,
|
31
|
-
# private key values
|
32
|
-
/(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)/,
|
33
33
|
# cred in http dump
|
34
34
|
/(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
|
35
35
|
].freeze
|
data/lib/aspera/ssh.rb
CHANGED
@@ -1,33 +1,41 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'net/ssh'
|
4
|
-
|
5
|
-
|
6
|
-
# HACK: deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
|
7
|
-
old_verbose = $VERBOSE
|
8
|
-
$VERBOSE = nil
|
9
|
-
begin
|
10
|
-
module Net; module SSH; module Authentication; class Session; private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa]; end; end; end; end; end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength, Style/Semicolon
|
11
|
-
rescue StandardError
|
12
|
-
# ignore errors
|
13
|
-
end
|
14
|
-
$VERBOSE = old_verbose
|
15
|
-
end
|
16
|
-
|
17
|
-
if defined?(JRUBY_VERSION) && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')
|
18
|
-
Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a =~ /^ecd(sa|h)-sha2/ } }
|
19
|
-
Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t =~ /^ecd(sa|h)-sha2/ }
|
20
|
-
end
|
4
|
+
require 'aspera/assert'
|
5
|
+
require 'aspera/log'
|
21
6
|
|
22
7
|
module Aspera
|
23
8
|
# A simple wrapper around Net::SSH
|
24
9
|
# executes one command and get its result from stdout
|
25
10
|
class Ssh
|
11
|
+
class << self
|
12
|
+
def disable_ed25519_keys
|
13
|
+
Log.log.debug('Disabling SSH ed25519 user keys')
|
14
|
+
old_verbose = $VERBOSE
|
15
|
+
$VERBOSE = nil
|
16
|
+
Net::SSH::Authentication::Session.class_eval do
|
17
|
+
define_method(:default_keys) do
|
18
|
+
%w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa].freeze
|
19
|
+
end
|
20
|
+
private(:default_keys)
|
21
|
+
end rescue nil
|
22
|
+
$VERBOSE = old_verbose
|
23
|
+
end
|
24
|
+
|
25
|
+
def disable_ecd_sha2_algorithms
|
26
|
+
Log.log.debug('Disabling SSH ecdsa')
|
27
|
+
Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a =~ /^ecd(sa|h)-sha2/ } }
|
28
|
+
Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t =~ /^ecd(sa|h)-sha2/ }
|
29
|
+
end
|
30
|
+
end
|
26
31
|
# ssh_options: same as Net::SSH.start
|
27
32
|
# see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
|
28
33
|
def initialize(host, username, ssh_options)
|
29
34
|
Log.log.debug{"ssh:#{username}@#{host}"}
|
30
35
|
Log.log.debug{"ssh_options:#{ssh_options}"}
|
36
|
+
Aspera.assert_type(host, String)
|
37
|
+
Aspera.assert_type(username, String)
|
38
|
+
Aspera.assert_type(ssh_options, Hash)
|
31
39
|
@host = host
|
32
40
|
@username = username
|
33
41
|
@ssh_options = ssh_options
|
@@ -35,10 +43,7 @@ module Aspera
|
|
35
43
|
end
|
36
44
|
|
37
45
|
def execute(cmd, input=nil)
|
38
|
-
|
39
|
-
# concatenate arguments, enclose in double quotes
|
40
|
-
cmd = cmd.map{|v|%Q("#{v}")}.join(' ')
|
41
|
-
end
|
46
|
+
Aspera.assert_type(cmd, String)
|
42
47
|
Log.log.debug{"cmd=#{cmd}"}
|
43
48
|
response = []
|
44
49
|
Net::SSH.start(@host, @username, @ssh_options) do |session|
|
@@ -49,9 +54,7 @@ module Aspera
|
|
49
54
|
channel.on_extended_data do |_chan, _type, data|
|
50
55
|
error_message = "#{cmd}: [#{data.chomp}]"
|
51
56
|
# Happens when windows user hasn't logged in and created home account.
|
52
|
-
if data.include?('Could not chdir to home directory')
|
53
|
-
error_message += "\nHint: home not created in Windows?"
|
54
|
-
end
|
57
|
+
error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
|
55
58
|
raise error_message
|
56
59
|
end
|
57
60
|
# send command to SSH channel (execute) cspell: disable-next-line
|
@@ -67,3 +70,7 @@ module Aspera
|
|
67
70
|
end
|
68
71
|
end
|
69
72
|
end
|
73
|
+
|
74
|
+
# HACK: deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
|
75
|
+
Aspera::Ssh.disable_ed25519_keys if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
|
76
|
+
Aspera::Ssh.disable_ecd_sha2_algorithms if defined?(JRUBY_VERSION) && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')
|
@@ -8,6 +8,7 @@ require 'aspera/transfer/error'
|
|
8
8
|
require 'aspera/transfer/spec'
|
9
9
|
require 'aspera/ascp/installation'
|
10
10
|
require 'aspera/cli/formatter'
|
11
|
+
require 'aspera/agent/base'
|
11
12
|
require 'aspera/rest'
|
12
13
|
require 'securerandom'
|
13
14
|
require 'base64'
|
@@ -20,7 +21,7 @@ module Aspera
|
|
20
21
|
# translate transfer specification to ascp parameter list
|
21
22
|
class Parameters
|
22
23
|
# Agents shown in manual for parameters (sub list)
|
23
|
-
SUPPORTED_AGENTS =
|
24
|
+
SUPPORTED_AGENTS = Agent::Base.agent_list.freeze
|
24
25
|
FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
|
25
26
|
# Short names of columns in manual
|
26
27
|
SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|agent_sym|agent_sym.to_s[0].to_sym}
|