aspera-cli 4.23.0 → 4.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +37 -1
- data/CONTRIBUTING.md +86 -29
- data/README.md +2109 -1300
- data/bin/ascli +2 -1
- data/bin/asession +4 -4
- data/lib/aspera/agent/base.rb +4 -0
- data/lib/aspera/agent/connect.rb +20 -18
- data/lib/aspera/agent/desktop.rb +14 -11
- data/lib/aspera/agent/direct.rb +39 -31
- data/lib/aspera/agent/httpgw.rb +2 -2
- data/lib/aspera/agent/node.rb +9 -11
- data/lib/aspera/agent/transferd.rb +18 -11
- data/lib/aspera/api/aoc.rb +44 -31
- data/lib/aspera/api/cos_node.rb +7 -5
- data/lib/aspera/api/httpgw.rb +15 -18
- data/lib/aspera/api/node.rb +104 -22
- data/lib/aspera/ascmd.rb +22 -16
- data/lib/aspera/ascp/installation.rb +37 -40
- data/lib/aspera/ascp/management.rb +5 -4
- data/lib/aspera/assert.rb +54 -23
- data/lib/aspera/cli/basic_auth_plugin.rb +8 -7
- data/lib/aspera/cli/error.rb +1 -1
- data/lib/aspera/cli/extended_value.rb +28 -29
- data/lib/aspera/cli/formatter.rb +191 -168
- data/lib/aspera/cli/hints.rb +29 -3
- data/lib/aspera/cli/main.rb +138 -107
- data/lib/aspera/cli/manager.rb +50 -30
- data/lib/aspera/cli/plugin.rb +148 -77
- data/lib/aspera/cli/plugin_factory.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +189 -70
- data/lib/aspera/cli/plugins/ats.rb +15 -13
- data/lib/aspera/cli/plugins/config.rb +100 -214
- data/lib/aspera/cli/plugins/console.rb +49 -18
- data/lib/aspera/cli/plugins/cos.rb +4 -4
- data/lib/aspera/cli/plugins/faspex.rb +45 -51
- data/lib/aspera/cli/plugins/faspex5.rb +164 -165
- data/lib/aspera/cli/plugins/faspio.rb +6 -5
- data/lib/aspera/cli/plugins/httpgw.rb +2 -2
- data/lib/aspera/cli/plugins/node.rb +144 -162
- data/lib/aspera/cli/plugins/orchestrator.rb +10 -14
- data/lib/aspera/cli/plugins/preview.rb +26 -29
- data/lib/aspera/cli/plugins/server.rb +28 -28
- data/lib/aspera/cli/plugins/shares.rb +40 -28
- data/lib/aspera/cli/sync_actions.rb +101 -80
- data/lib/aspera/cli/transfer_agent.rb +51 -50
- data/lib/aspera/cli/transfer_progress.rb +29 -20
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +157 -0
- data/lib/aspera/colors.rb +13 -8
- data/lib/aspera/command_line_builder.rb +28 -22
- data/lib/aspera/command_line_converter.rb +31 -0
- data/lib/aspera/environment.rb +145 -101
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/faspex_postproc.rb +3 -2
- data/lib/aspera/hash_ext.rb +1 -1
- data/lib/aspera/id_generator.rb +10 -10
- data/lib/aspera/keychain/base.rb +18 -0
- data/lib/aspera/keychain/encrypted_hash.rb +6 -12
- data/lib/aspera/keychain/factory.rb +9 -3
- data/lib/aspera/keychain/hashicorp_vault.rb +9 -6
- data/lib/aspera/keychain/macos_security.rb +13 -13
- data/lib/aspera/log.rb +91 -19
- data/lib/aspera/nagios.rb +5 -6
- data/lib/aspera/node_simulator.rb +12 -7
- data/lib/aspera/oauth/base.rb +5 -3
- data/lib/aspera/oauth/factory.rb +24 -18
- data/lib/aspera/oauth/jwt.rb +13 -1
- data/lib/aspera/oauth/url_json.rb +3 -3
- data/lib/aspera/oauth/web.rb +5 -3
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +4 -3
- data/lib/aspera/preview/generator.rb +25 -12
- data/lib/aspera/preview/terminal.rb +10 -7
- data/lib/aspera/preview/utils.rb +11 -9
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/desktop.rb +1 -1
- data/lib/aspera/products/other.rb +2 -2
- data/lib/aspera/products/transferd.rb +8 -6
- data/lib/aspera/proxy_auto_config.rb +1 -1
- data/lib/aspera/rest.rb +29 -22
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/resumer.rb +1 -1
- data/lib/aspera/secret_hider.rb +46 -40
- data/lib/aspera/ssh.rb +13 -3
- data/lib/aspera/sync/args.schema.yaml +102 -0
- data/lib/aspera/sync/conf.schema.yaml +701 -0
- data/lib/aspera/sync/database.rb +83 -0
- data/lib/aspera/sync/operations.rb +296 -0
- data/lib/aspera/temp_file_manager.rb +3 -2
- data/lib/aspera/transfer/error.rb +1 -1
- data/lib/aspera/transfer/error_info.rb +1 -2
- data/lib/aspera/transfer/faux_file.rb +11 -10
- data/lib/aspera/transfer/parameters.rb +6 -5
- data/lib/aspera/transfer/spec.rb +15 -1
- data/lib/aspera/transfer/spec.schema.yaml +316 -293
- data/lib/aspera/transfer/spec_doc.rb +34 -16
- data/lib/aspera/transfer/uri.rb +5 -5
- data/lib/aspera/uri_reader.rb +14 -10
- data/lib/aspera/web_auth.rb +2 -2
- data/lib/aspera/web_server_simple.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +15 -13
- metadata.gz.sig +0 -0
- data/lib/aspera/transfer/async_conf.schema.yaml +0 -716
- data/lib/aspera/transfer/convert.rb +0 -29
- data/lib/aspera/transfer/sync.rb +0 -232
- data/lib/aspera/transfer/sync_instance.schema.yaml +0 -20
- data/lib/aspera/transfer/sync_session.schema.yaml +0 -86
@@ -19,10 +19,10 @@ module Aspera
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def create_token
|
22
|
-
|
22
|
+
api.call(
|
23
23
|
operation: 'POST',
|
24
|
-
subpath:
|
25
|
-
query: @query.merge(scope:
|
24
|
+
subpath: path_token,
|
25
|
+
query: @query.merge(scope: scope), # scope is here because it may change over time (node)
|
26
26
|
content_type: Rest::MIME_JSON,
|
27
27
|
body: @body,
|
28
28
|
headers: {'Accept' => Rest::MIME_JSON}
|
data/lib/aspera/oauth/web.rb
CHANGED
@@ -31,8 +31,9 @@ module Aspera
|
|
31
31
|
# generate secure state to check later
|
32
32
|
random_state = SecureRandom.uuid
|
33
33
|
login_page_url = Rest.build_uri(
|
34
|
-
"#{
|
35
|
-
optional_scope_client_id.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state)
|
34
|
+
"#{api.base_url}/#{@path_authorize}",
|
35
|
+
optional_scope_client_id.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state)
|
36
|
+
)
|
36
37
|
# here, we need a human to authorize on a web page
|
37
38
|
Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
|
38
39
|
# start a web server to receive request code
|
@@ -46,7 +47,8 @@ module Aspera
|
|
46
47
|
return create_token_call(optional_scope_client_id(add_secret: true).merge(
|
47
48
|
grant_type: 'authorization_code',
|
48
49
|
code: received_params['code'],
|
49
|
-
redirect_uri: @redirect_uri
|
50
|
+
redirect_uri: @redirect_uri
|
51
|
+
))
|
50
52
|
end
|
51
53
|
end
|
52
54
|
Factory.instance.register_token_creator(Web)
|
@@ -60,7 +60,7 @@ module Aspera
|
|
60
60
|
end
|
61
61
|
|
62
62
|
# Delete persisted items
|
63
|
-
def garbage_collect(persist_category, max_age_seconds=nil)
|
63
|
+
def garbage_collect(persist_category, max_age_seconds = nil)
|
64
64
|
garbage_files = current_files(persist_category)
|
65
65
|
if !max_age_seconds.nil?
|
66
66
|
current_time = Time.now
|
@@ -75,7 +75,7 @@ module Aspera
|
|
75
75
|
end
|
76
76
|
|
77
77
|
def current_files(persist_category)
|
78
|
-
Dir[File.join(@folder, persist_category
|
78
|
+
Dir[File.join(@folder, "#{persist_category}*#{FILE_SUFFIX}")]
|
79
79
|
end
|
80
80
|
|
81
81
|
def current_items(persist_category)
|
@@ -48,7 +48,8 @@ module Aspera
|
|
48
48
|
'application/x-xfig' => :image,
|
49
49
|
'font/ttf' => :image,
|
50
50
|
'text/troff' => :image,
|
51
|
-
'video/x-mng' => :image
|
51
|
+
'video/x-mng' => :image
|
52
|
+
}.freeze
|
52
53
|
|
53
54
|
private_constant :SUPPORTED_MIME_TYPES
|
54
55
|
|
@@ -68,7 +69,7 @@ module Aspera
|
|
68
69
|
return :office if mimetype.start_with?('application/vnd.openxmlformats-officedocument')
|
69
70
|
return :video if mimetype.start_with?('video/')
|
70
71
|
return :image if mimetype.start_with?('image/')
|
71
|
-
return
|
72
|
+
return
|
72
73
|
end
|
73
74
|
|
74
75
|
# @param filepath [String] full path to file
|
@@ -118,7 +119,7 @@ module Aspera
|
|
118
119
|
return Environment.secure_capture(exec: 'file', args: ['--mime-type', '--brief', filepath]).strip
|
119
120
|
rescue => e
|
120
121
|
Log.log.error{"error using 'file' command: #{e.message}"}
|
121
|
-
return
|
122
|
+
return
|
122
123
|
end
|
123
124
|
end
|
124
125
|
end
|
@@ -116,7 +116,9 @@ module Aspera
|
|
116
116
|
'-filter:v', "scale='trunc(iw/2)*2:trunc(ih/2)*2'",
|
117
117
|
'-codec:v', 'libx264',
|
118
118
|
'-r', 30,
|
119
|
-
'-pix_fmt', 'yuv420p'
|
119
|
+
'-pix_fmt', 'yuv420p'
|
120
|
+
]
|
121
|
+
)
|
120
122
|
end
|
121
123
|
|
122
124
|
# generate n clips starting at offset
|
@@ -135,7 +137,9 @@ module Aspera
|
|
135
137
|
'-ss', offset_seconds * 0.1,
|
136
138
|
'-t', @options.clips_length,
|
137
139
|
'-filter:v', "scale=#{@options.video_scale}",
|
138
|
-
'-codec:a', 'libmp3lame'
|
140
|
+
'-codec:a', 'libmp3lame'
|
141
|
+
]
|
142
|
+
)
|
139
143
|
f.puts("file '#{tmp_file_name}'")
|
140
144
|
end
|
141
145
|
end
|
@@ -144,7 +148,8 @@ module Aspera
|
|
144
148
|
in_f: file_list_file,
|
145
149
|
in_p: ['-f', 'concat'],
|
146
150
|
out_f: @destination_file_path,
|
147
|
-
out_p: ['-codec', 'copy']
|
151
|
+
out_p: ['-codec', 'copy']
|
152
|
+
)
|
148
153
|
File.delete(file_list_file)
|
149
154
|
end
|
150
155
|
|
@@ -174,7 +179,9 @@ module Aspera
|
|
174
179
|
'-codec:a', 'libmp3lame',
|
175
180
|
'-ac', '2',
|
176
181
|
'-b:a', '128k',
|
177
|
-
'-movflags', 'faststart'
|
182
|
+
'-movflags', 'faststart'
|
183
|
+
]
|
184
|
+
)
|
178
185
|
end
|
179
186
|
|
180
187
|
def convert_video_to_png_using_fixed
|
@@ -182,7 +189,8 @@ module Aspera
|
|
182
189
|
@source_file_path,
|
183
190
|
Utils.video_get_duration(@source_file_path) * @options.thumb_vid_fraction,
|
184
191
|
@options.thumb_vid_scale,
|
185
|
-
@destination_file_path
|
192
|
+
@destination_file_path
|
193
|
+
)
|
186
194
|
end
|
187
195
|
|
188
196
|
# https://trac.ffmpeg.org/wiki/SponsoringPrograms/GSoC/2015#AnimatedPortableNetworkGraphicsAPNG
|
@@ -201,19 +209,21 @@ module Aspera
|
|
201
209
|
'-vf', 'fps=5,scale=120:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse',
|
202
210
|
'-loop', 0,
|
203
211
|
'-f', 'gif'
|
204
|
-
]
|
212
|
+
]
|
213
|
+
)
|
205
214
|
end
|
206
215
|
|
207
216
|
def convert_office_to_png
|
208
|
-
tmp_pdf_file = File.join(this_tmpdir, File.basename(@source_file_path, File.extname(@source_file_path))
|
217
|
+
tmp_pdf_file = File.join(this_tmpdir, "#{File.basename(@source_file_path, File.extname(@source_file_path))}.pdf")
|
209
218
|
Utils.external_command(:unoconv, [
|
210
219
|
'-f', 'pdf',
|
211
220
|
'-o', tmp_pdf_file,
|
212
|
-
@source_file_path
|
221
|
+
@source_file_path
|
222
|
+
])
|
213
223
|
convert_pdf_to_png(tmp_pdf_file)
|
214
224
|
end
|
215
225
|
|
216
|
-
def convert_pdf_to_png(source_file_path=nil)
|
226
|
+
def convert_pdf_to_png(source_file_path = nil)
|
217
227
|
source_file_path ||= @source_file_path
|
218
228
|
Utils.external_command(:magick, [
|
219
229
|
'convert',
|
@@ -221,7 +231,8 @@ module Aspera
|
|
221
231
|
'-background', 'white',
|
222
232
|
'-flatten',
|
223
233
|
"#{source_file_path}[0]",
|
224
|
-
@destination_file_path
|
234
|
+
@destination_file_path
|
235
|
+
])
|
225
236
|
end
|
226
237
|
|
227
238
|
def convert_image_to_png
|
@@ -233,7 +244,8 @@ module Aspera
|
|
233
244
|
'+dither',
|
234
245
|
'-posterize', 40,
|
235
246
|
"#{@source_file_path}[0]",
|
236
|
-
@destination_file_path
|
247
|
+
@destination_file_path
|
248
|
+
])
|
237
249
|
Utils.external_command(:optipng, [@destination_file_path])
|
238
250
|
end
|
239
251
|
|
@@ -253,7 +265,8 @@ module Aspera
|
|
253
265
|
'-bordercolor', 'white',
|
254
266
|
'-border', 8,
|
255
267
|
'+repage',
|
256
|
-
@destination_file_path
|
268
|
+
@destination_file_path
|
269
|
+
])
|
257
270
|
end
|
258
271
|
end
|
259
272
|
end
|
@@ -5,9 +5,11 @@
|
|
5
5
|
require 'rainbow'
|
6
6
|
require 'io/console'
|
7
7
|
require 'aspera/log'
|
8
|
+
require 'aspera/environment'
|
8
9
|
module Aspera
|
9
10
|
module Preview
|
10
|
-
# Display a picture in the terminal
|
11
|
+
# Display a picture in the terminal.
|
12
|
+
# Either use coloured characters or iTerm2 protocol.
|
11
13
|
class Terminal
|
12
14
|
# Rainbow only supports 8-bit colors
|
13
15
|
# env vars to detect terminal type
|
@@ -19,13 +21,14 @@ module Aspera
|
|
19
21
|
DEFAULT_FONT_RATIO = 32.0 / 14.0
|
20
22
|
private_constant :TERM_ENV_VARS, :ITERM_NAMES, :DEFAULT_FONT_RATIO
|
21
23
|
class << self
|
22
|
-
# @
|
23
|
-
# @param
|
24
|
-
# @param
|
25
|
-
# @param
|
26
|
-
# @param
|
27
|
-
# @
|
24
|
+
# @param blob [String] The image as a binary string
|
25
|
+
# @param reserve [Integer] Number of lines to reserve for other text than the image
|
26
|
+
# @param text [Boolean] `true` to display the image as text, `false` to use iTerm2 if supported
|
27
|
+
# @param double [Boolean] `true` to use colors on half lines, `false` to use colors on full lines
|
28
|
+
# @param font_ratio [Float] ratio = font height / font width
|
29
|
+
# @return [String] The image as text, or the iTerm2 escape sequence
|
28
30
|
def build(blob, reserve: 3, text: false, double: true, font_ratio: DEFAULT_FONT_RATIO)
|
31
|
+
return '[Image display requires a terminal]' unless Environment.terminal?
|
29
32
|
return iterm_display_image(blob) if iterm_supported? && !text
|
30
33
|
begin
|
31
34
|
# do not require statically, as the package is optional
|
data/lib/aspera/preview/utils.rb
CHANGED
@@ -31,7 +31,7 @@ module Aspera
|
|
31
31
|
end
|
32
32
|
|
33
33
|
# check that external tools can be executed
|
34
|
-
def check_tools(skip_types=[])
|
34
|
+
def check_tools(skip_types = [])
|
35
35
|
tools_to_check = EXTERNAL_TOOLS.dup
|
36
36
|
tools_to_check.delete(:unoconv) if skip_types.include?(:office)
|
37
37
|
# Check for binaries
|
@@ -70,7 +70,8 @@ module Aspera
|
|
70
70
|
'-loglevel', 'error',
|
71
71
|
'-show_entries', 'format=duration',
|
72
72
|
'-print_format', 'default=noprint_wrappers=1:nokey=1', # cspell:disable-line
|
73
|
-
input_file
|
73
|
+
input_file
|
74
|
+
]).to_f
|
74
75
|
end
|
75
76
|
|
76
77
|
def ffmpeg_fmt(temp_folder)
|
@@ -88,24 +89,25 @@ module Aspera
|
|
88
89
|
end
|
89
90
|
end
|
90
91
|
|
91
|
-
def video_blend_frames(temp_folder,
|
92
|
-
img1 = get_tmp_num_filepath(temp_folder,
|
93
|
-
img2 = get_tmp_num_filepath(temp_folder,
|
94
|
-
count =
|
92
|
+
def video_blend_frames(temp_folder, index_begin, index_end)
|
93
|
+
img1 = get_tmp_num_filepath(temp_folder, index_begin)
|
94
|
+
img2 = get_tmp_num_filepath(temp_folder, index_end)
|
95
|
+
count = index_end - index_begin - 1
|
95
96
|
1.upto(count) do |i|
|
96
97
|
percent = i * 100 / (count + 1)
|
97
|
-
filename = get_tmp_num_filepath(temp_folder,
|
98
|
+
filename = get_tmp_num_filepath(temp_folder, index_begin + i)
|
98
99
|
external_command(:magick, ['composite', '-blend', percent, img2, img1, filename])
|
99
100
|
end
|
100
101
|
end
|
101
102
|
|
102
|
-
def video_dump_frame(input_file, offset_seconds, scale, output_file, index=nil)
|
103
|
+
def video_dump_frame(input_file, offset_seconds, scale, output_file, index = nil)
|
103
104
|
output_file = get_tmp_num_filepath(output_file, index) unless index.nil?
|
104
105
|
ffmpeg(
|
105
106
|
in_f: input_file,
|
106
107
|
in_p: ['-ss', offset_seconds],
|
107
108
|
out_f: output_file,
|
108
|
-
out_p: ['-frames:v', 1, '-filter:v', "scale=#{scale}"]
|
109
|
+
out_p: ['-frames:v', 1, '-filter:v', "scale=#{scale}"]
|
110
|
+
)
|
109
111
|
return output_file
|
110
112
|
end
|
111
113
|
end
|
@@ -13,7 +13,7 @@ module Aspera
|
|
13
13
|
class << self
|
14
14
|
# standard folder locations
|
15
15
|
def locations
|
16
|
-
case Aspera::Environment.os
|
16
|
+
case Aspera::Environment.instance.os
|
17
17
|
when Aspera::Environment::OS_WINDOWS then [{
|
18
18
|
app_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Programs', 'Aspera', 'Aspera Connect'),
|
19
19
|
log_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Aspera', 'Aspera Connect', 'var', 'log'),
|
@@ -11,7 +11,7 @@ module Aspera
|
|
11
11
|
class << self
|
12
12
|
# standard folder locations
|
13
13
|
def locations
|
14
|
-
case Aspera::Environment.os
|
14
|
+
case Aspera::Environment.instance.os
|
15
15
|
when Aspera::Environment::OS_MACOS then [{
|
16
16
|
app_root: File.join('', 'Applications', 'IBM Aspera.app'),
|
17
17
|
log_root: File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER),
|
@@ -21,7 +21,7 @@ module Aspera
|
|
21
21
|
# :log_root O location of log files (Linux uses syslog)
|
22
22
|
# :run_root O only for Connect Client, location of http port file
|
23
23
|
# :sub_bin O subfolder with executables, default : bin
|
24
|
-
LOCATION_ON_THIS_OS = case Aspera::Environment.os
|
24
|
+
LOCATION_ON_THIS_OS = case Aspera::Environment.instance.os
|
25
25
|
when Aspera::Environment::OS_WINDOWS then [{
|
26
26
|
expected: CLI_V3,
|
27
27
|
app_root: File.join('C:', 'Program Files', 'Aspera', 'cli'),
|
@@ -61,7 +61,7 @@ module Aspera
|
|
61
61
|
next false unless Dir.exist?(item[:app_root])
|
62
62
|
Log.log.debug{"Found #{item[:expected]}"}
|
63
63
|
sub_bin = item[:sub_bin] || 'bin'
|
64
|
-
item[:ascp_path] = File.join(item[:app_root], sub_bin, Environment.exe_file('ascp'))
|
64
|
+
item[:ascp_path] = File.join(item[:app_root], sub_bin, Environment.instance.exe_file('ascp'))
|
65
65
|
# skip if no ascp
|
66
66
|
next false unless File.exist?(item[:ascp_path])
|
67
67
|
# read info from product info file if present
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
3
5
|
module Aspera
|
4
6
|
module Products
|
5
7
|
class Transferd
|
@@ -19,9 +21,9 @@ module Aspera
|
|
19
21
|
end
|
20
22
|
|
21
23
|
# location of SDK files
|
22
|
-
def sdk_directory=(
|
23
|
-
Log.log.debug{"sdk_directory=#{
|
24
|
-
@sdk_dir =
|
24
|
+
def sdk_directory=(folder)
|
25
|
+
Log.log.debug{"sdk_directory=#{folder}"}
|
26
|
+
@sdk_dir = folder
|
25
27
|
sdk_directory
|
26
28
|
end
|
27
29
|
|
@@ -33,9 +35,9 @@ module Aspera
|
|
33
35
|
end
|
34
36
|
|
35
37
|
def transferd_path
|
36
|
-
v1_path = File.join(sdk_directory, Environment.exe_file(V1_DAEMON_NAME))
|
38
|
+
v1_path = File.join(sdk_directory, Environment.instance.exe_file(V1_DAEMON_NAME))
|
37
39
|
return v1_path if File.exist?(v1_path)
|
38
|
-
return File.join(sdk_directory, Environment.exe_file(V2_DAEMON_NAME))
|
40
|
+
return File.join(sdk_directory, Environment.instance.exe_file(V2_DAEMON_NAME))
|
39
41
|
end
|
40
42
|
|
41
43
|
# Well, the port number is only in log file
|
@@ -51,7 +53,7 @@ module Aspera
|
|
51
53
|
end
|
52
54
|
end
|
53
55
|
end
|
54
|
-
|
56
|
+
Aspera.assert(!result.nil?){'Port not found in daemon logs'}
|
55
57
|
Log.log.debug{"Got port #{result} from log"}
|
56
58
|
return result
|
57
59
|
end
|
@@ -13,7 +13,7 @@ module URI
|
|
13
13
|
def register_proxy_finder
|
14
14
|
Aspera.assert(block_given?)
|
15
15
|
# overload the method in URI : call user's provided block and fallback to original method
|
16
|
-
define_method(:find_proxy){ |env_vars=ENV| yield(to_s) || find_proxy_orig(env_vars)}
|
16
|
+
define_method(:find_proxy){ |env_vars = ENV| yield(to_s) || find_proxy_orig(env_vars)}
|
17
17
|
end
|
18
18
|
end
|
19
19
|
end
|
data/lib/aspera/rest.rb
CHANGED
@@ -88,11 +88,11 @@ module Aspera
|
|
88
88
|
# Build URI from URL and parameters and check it is http or https
|
89
89
|
# encode array [] parameters
|
90
90
|
# @param query [Hash,Array]
|
91
|
-
def build_uri(url, query=nil)
|
91
|
+
def build_uri(url, query = nil)
|
92
92
|
uri = URI.parse(url)
|
93
93
|
Aspera.assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
|
94
94
|
return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
|
95
|
-
Log.
|
95
|
+
Log.dump(:query, query)
|
96
96
|
query_array = []
|
97
97
|
case query
|
98
98
|
when Hash
|
@@ -194,9 +194,7 @@ module Aspera
|
|
194
194
|
|
195
195
|
# create and start keep alive connection on demand
|
196
196
|
def http_session
|
197
|
-
if @http_session.nil?
|
198
|
-
@http_session = self.class.start_http_session(@base_url)
|
199
|
-
end
|
197
|
+
@http_session = self.class.start_http_session(@base_url) if @http_session.nil?
|
200
198
|
return @http_session
|
201
199
|
end
|
202
200
|
|
@@ -239,7 +237,7 @@ module Aspera
|
|
239
237
|
)
|
240
238
|
Aspera.assert_type(base_url, String)
|
241
239
|
# base url with no trailing slashes (note: string may be frozen)
|
242
|
-
@base_url = base_url.
|
240
|
+
@base_url = base_url.chomp('/')
|
243
241
|
# remove trailing port if it is 443 and scheme is https
|
244
242
|
@base_url = @base_url.gsub(/:443$/, '') if @base_url.start_with?('https://')
|
245
243
|
@base_url = @base_url.gsub(/:80$/, '') if @base_url.start_with?('http://')
|
@@ -266,7 +264,7 @@ module Aspera
|
|
266
264
|
if @oauth.nil?
|
267
265
|
Aspera.assert(@auth_params[:type].eql?(:oauth2)){'no OAuth defined'}
|
268
266
|
oauth_parameters = @auth_params.reject{ |k, _v| k.eql?(:type)}
|
269
|
-
Log.
|
267
|
+
Log.dump(:oauth_parameters, oauth_parameters)
|
270
268
|
@oauth = OAuth::Factory.instance.create(**oauth_parameters)
|
271
269
|
end
|
272
270
|
return @oauth
|
@@ -293,8 +291,8 @@ module Aspera
|
|
293
291
|
)
|
294
292
|
subpath = subpath.to_s if subpath.is_a?(Symbol)
|
295
293
|
subpath = '' if subpath.nil?
|
296
|
-
Log.log.debug{"#{operation} [#{subpath}]".red.bold.bg_green}
|
297
|
-
Log.
|
294
|
+
Log.log.debug{"call #{operation} [#{subpath}]".red.bold.bg_green}
|
295
|
+
Log.dump(:body, body)
|
298
296
|
Aspera.assert_type(subpath, String)
|
299
297
|
if headers.nil?
|
300
298
|
headers = @headers.clone
|
@@ -352,7 +350,7 @@ module Aspera
|
|
352
350
|
end
|
353
351
|
# :type = :basic
|
354
352
|
req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
|
355
|
-
Log.
|
353
|
+
Log.dump(:req_body, req.body, level: :trace1)
|
356
354
|
# we try the call, and will retry on some error types
|
357
355
|
error_tries ||= 1 + RestParameters.instance.retry_max
|
358
356
|
# initialize with number of initial retries allowed, nil gives zero
|
@@ -375,9 +373,7 @@ module Aspera
|
|
375
373
|
# override user's path to path in header
|
376
374
|
if !response['Content-Disposition'].nil?
|
377
375
|
disposition = self.class.parse_header(response['Content-Disposition'])
|
378
|
-
if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
|
379
|
-
target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename])
|
380
|
-
end
|
376
|
+
target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename]) if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
|
381
377
|
end
|
382
378
|
# download with temp filename
|
383
379
|
target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
|
@@ -395,7 +391,8 @@ module Aspera
|
|
395
391
|
RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
|
396
392
|
end
|
397
393
|
end
|
398
|
-
RestParameters.instance.progress_bar&.event(:
|
394
|
+
RestParameters.instance.progress_bar&.event(:session_end, session_id: session_id)
|
395
|
+
RestParameters.instance.progress_bar&.event(:end)
|
399
396
|
# rename at the end
|
400
397
|
File.rename(target_file_tmp, target_file)
|
401
398
|
file_saved = true
|
@@ -408,7 +405,7 @@ module Aspera
|
|
408
405
|
case result_mime
|
409
406
|
when *JSON_DECODE
|
410
407
|
result[:data] = JSON.parse(result[:http].body) rescue result[:http].body
|
411
|
-
Log.
|
408
|
+
Log.dump(:result_data, result[:data])
|
412
409
|
else # when MIME_TEXT
|
413
410
|
result[:data] = result[:http].body
|
414
411
|
end
|
@@ -447,7 +444,7 @@ module Aspera
|
|
447
444
|
if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
|
448
445
|
tries_remain_redirect -= 1
|
449
446
|
current_uri = URI.parse(@base_url)
|
450
|
-
new_url = e.response['
|
447
|
+
new_url = e.response['Location']
|
451
448
|
# special case: relative redirect
|
452
449
|
if URI.parse(new_url).host.nil?
|
453
450
|
# we don't manage relative redirects with non-absolute path
|
@@ -455,9 +452,19 @@ module Aspera
|
|
455
452
|
new_url = "#{current_uri.scheme}://#{current_uri.host}#{new_url}"
|
456
453
|
end
|
457
454
|
# forwards the request to the new location
|
458
|
-
return self.class.new(
|
459
|
-
|
460
|
-
|
455
|
+
return self.class.new(
|
456
|
+
base_url: new_url,
|
457
|
+
redirect_max: tries_remain_redirect
|
458
|
+
).call(
|
459
|
+
operation: operation,
|
460
|
+
subpath: new_url.end_with?('/') ? '/' : nil,
|
461
|
+
query: query,
|
462
|
+
body: body,
|
463
|
+
content_type: content_type,
|
464
|
+
save_to_file: save_to_file,
|
465
|
+
return_error: return_error,
|
466
|
+
headers: headers
|
467
|
+
)
|
461
468
|
end
|
462
469
|
# raise exception if could not retry and not return error in result
|
463
470
|
raise e unless return_error
|
@@ -475,7 +482,7 @@ module Aspera
|
|
475
482
|
return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
|
476
483
|
end
|
477
484
|
|
478
|
-
def read(subpath, query=nil)
|
485
|
+
def read(subpath, query = nil)
|
479
486
|
return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query)[:data]
|
480
487
|
end
|
481
488
|
|
@@ -483,7 +490,7 @@ module Aspera
|
|
483
490
|
return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
|
484
491
|
end
|
485
492
|
|
486
|
-
def delete(subpath, params=nil)
|
493
|
+
def delete(subpath, params = nil)
|
487
494
|
return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params)[:data]
|
488
495
|
end
|
489
496
|
|
@@ -513,7 +520,7 @@ module Aspera
|
|
513
520
|
name_matches = matching_items.select{ |i| i['name'].casecmp?(search_name)}
|
514
521
|
case name_matches.length
|
515
522
|
when 1 then return name_matches.first
|
516
|
-
when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
|
523
|
+
when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
|
517
524
|
else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{ |i| i['name']}}"
|
518
525
|
end
|
519
526
|
end
|
data/lib/aspera/resumer.rb
CHANGED
data/lib/aspera/secret_hider.rb
CHANGED
@@ -2,10 +2,13 @@
|
|
2
2
|
|
3
3
|
# cspell:ignore FILEPASS
|
4
4
|
require 'logger'
|
5
|
+
require 'singleton'
|
5
6
|
|
6
7
|
module Aspera
|
7
8
|
# remove secret from logs and output
|
8
9
|
class SecretHider
|
10
|
+
include Singleton
|
11
|
+
|
9
12
|
# configurable:
|
10
13
|
ADDITIONAL_KEYS_TO_HIDE = []
|
11
14
|
# display string for hidden secrets
|
@@ -34,56 +37,59 @@ module Aspera
|
|
34
37
|
/(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
|
35
38
|
].freeze
|
36
39
|
private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :KEY_FALSE_POSITIVES, :REGEX_LOG_REPLACES
|
37
|
-
|
38
|
-
class << self
|
39
|
-
attr_accessor :log_secrets
|
40
|
+
attr_accessor :log_secrets
|
40
41
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
end
|
42
|
+
# @return new log formatter that hides secrets
|
43
|
+
def log_formatter(original_formatter)
|
44
|
+
original_formatter ||= Logger::Formatter.new
|
45
|
+
# NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
|
46
|
+
return lambda do |severity, date_time, program_name, msg|
|
47
|
+
if msg.is_a?(String) && !@log_secrets
|
48
|
+
REGEX_LOG_REPLACES.each do |reg_ex|
|
49
|
+
msg = msg.gsub(reg_ex){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
|
50
50
|
end
|
51
|
-
original_formatter.call(severity, date_time, program_name, msg)
|
52
51
|
end
|
52
|
+
original_formatter.call(severity, date_time, program_name, msg)
|
53
53
|
end
|
54
|
+
end
|
54
55
|
|
55
|
-
|
56
|
-
|
57
|
-
|
56
|
+
def hide_secrets_in_string(value)
|
57
|
+
return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
|
58
|
+
end
|
58
59
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
60
|
+
# @return true if the key denotes a secret
|
61
|
+
def secret?(keyword, value)
|
62
|
+
keyword = keyword.to_s if keyword.is_a?(Symbol)
|
63
|
+
# only Strings can be secrets, not booleans, or hash, arrays
|
64
|
+
return false unless keyword.is_a?(String) && value.is_a?(String)
|
65
|
+
# those are not secrets
|
66
|
+
return false if KEY_FALSE_POSITIVES.any?{ |f| f.match?(keyword)}
|
67
|
+
return true if ADDITIONAL_KEYS_TO_HIDE.include?(keyword)
|
68
|
+
# check if keyword (name) contains an element that designate it as a secret
|
69
|
+
ALL_SECRETS.any?{ |kw| keyword.include?(kw)}
|
70
|
+
end
|
70
71
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
end
|
72
|
+
# Hides recursively secrets in Hash or Array of Hash
|
73
|
+
def deep_remove_secret(obj)
|
74
|
+
case obj
|
75
|
+
when Array
|
76
|
+
obj.each{ |i| deep_remove_secret(i)}
|
77
|
+
when Hash
|
78
|
+
obj.each do |k, v|
|
79
|
+
if secret?(k, v)
|
80
|
+
obj[k] = HIDDEN_PASSWORD
|
81
|
+
elsif obj[k].is_a?(Hash)
|
82
|
+
deep_remove_secret(obj[k])
|
83
83
|
end
|
84
84
|
end
|
85
|
-
return obj
|
86
85
|
end
|
86
|
+
return obj
|
87
|
+
end
|
88
|
+
|
89
|
+
private
|
90
|
+
|
91
|
+
def initialize
|
92
|
+
@log_secrets = false
|
87
93
|
end
|
88
94
|
end
|
89
95
|
end
|