aspera-cli 4.23.0 → 4.24.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +37 -1
  4. data/CONTRIBUTING.md +86 -29
  5. data/README.md +2109 -1300
  6. data/bin/ascli +2 -1
  7. data/bin/asession +4 -4
  8. data/lib/aspera/agent/base.rb +4 -0
  9. data/lib/aspera/agent/connect.rb +20 -18
  10. data/lib/aspera/agent/desktop.rb +14 -11
  11. data/lib/aspera/agent/direct.rb +39 -31
  12. data/lib/aspera/agent/httpgw.rb +2 -2
  13. data/lib/aspera/agent/node.rb +9 -11
  14. data/lib/aspera/agent/transferd.rb +18 -11
  15. data/lib/aspera/api/aoc.rb +44 -31
  16. data/lib/aspera/api/cos_node.rb +7 -5
  17. data/lib/aspera/api/httpgw.rb +15 -18
  18. data/lib/aspera/api/node.rb +104 -22
  19. data/lib/aspera/ascmd.rb +22 -16
  20. data/lib/aspera/ascp/installation.rb +37 -40
  21. data/lib/aspera/ascp/management.rb +5 -4
  22. data/lib/aspera/assert.rb +54 -23
  23. data/lib/aspera/cli/basic_auth_plugin.rb +8 -7
  24. data/lib/aspera/cli/error.rb +1 -1
  25. data/lib/aspera/cli/extended_value.rb +28 -29
  26. data/lib/aspera/cli/formatter.rb +191 -168
  27. data/lib/aspera/cli/hints.rb +29 -3
  28. data/lib/aspera/cli/main.rb +138 -107
  29. data/lib/aspera/cli/manager.rb +50 -30
  30. data/lib/aspera/cli/plugin.rb +148 -77
  31. data/lib/aspera/cli/plugin_factory.rb +2 -2
  32. data/lib/aspera/cli/plugins/aoc.rb +189 -70
  33. data/lib/aspera/cli/plugins/ats.rb +15 -13
  34. data/lib/aspera/cli/plugins/config.rb +100 -214
  35. data/lib/aspera/cli/plugins/console.rb +49 -18
  36. data/lib/aspera/cli/plugins/cos.rb +4 -4
  37. data/lib/aspera/cli/plugins/faspex.rb +45 -51
  38. data/lib/aspera/cli/plugins/faspex5.rb +164 -165
  39. data/lib/aspera/cli/plugins/faspio.rb +6 -5
  40. data/lib/aspera/cli/plugins/httpgw.rb +2 -2
  41. data/lib/aspera/cli/plugins/node.rb +144 -162
  42. data/lib/aspera/cli/plugins/orchestrator.rb +10 -14
  43. data/lib/aspera/cli/plugins/preview.rb +26 -29
  44. data/lib/aspera/cli/plugins/server.rb +28 -28
  45. data/lib/aspera/cli/plugins/shares.rb +40 -28
  46. data/lib/aspera/cli/sync_actions.rb +101 -80
  47. data/lib/aspera/cli/transfer_agent.rb +51 -50
  48. data/lib/aspera/cli/transfer_progress.rb +29 -20
  49. data/lib/aspera/cli/version.rb +1 -1
  50. data/lib/aspera/cli/wizard.rb +157 -0
  51. data/lib/aspera/colors.rb +13 -8
  52. data/lib/aspera/command_line_builder.rb +28 -22
  53. data/lib/aspera/command_line_converter.rb +31 -0
  54. data/lib/aspera/environment.rb +145 -101
  55. data/lib/aspera/faspex_gw.rb +1 -1
  56. data/lib/aspera/faspex_postproc.rb +3 -2
  57. data/lib/aspera/hash_ext.rb +1 -1
  58. data/lib/aspera/id_generator.rb +10 -10
  59. data/lib/aspera/keychain/base.rb +18 -0
  60. data/lib/aspera/keychain/encrypted_hash.rb +6 -12
  61. data/lib/aspera/keychain/factory.rb +9 -3
  62. data/lib/aspera/keychain/hashicorp_vault.rb +9 -6
  63. data/lib/aspera/keychain/macos_security.rb +13 -13
  64. data/lib/aspera/log.rb +91 -19
  65. data/lib/aspera/nagios.rb +5 -6
  66. data/lib/aspera/node_simulator.rb +12 -7
  67. data/lib/aspera/oauth/base.rb +5 -3
  68. data/lib/aspera/oauth/factory.rb +24 -18
  69. data/lib/aspera/oauth/jwt.rb +13 -1
  70. data/lib/aspera/oauth/url_json.rb +3 -3
  71. data/lib/aspera/oauth/web.rb +5 -3
  72. data/lib/aspera/persistency_folder.rb +2 -2
  73. data/lib/aspera/preview/file_types.rb +4 -3
  74. data/lib/aspera/preview/generator.rb +25 -12
  75. data/lib/aspera/preview/terminal.rb +10 -7
  76. data/lib/aspera/preview/utils.rb +11 -9
  77. data/lib/aspera/products/connect.rb +1 -1
  78. data/lib/aspera/products/desktop.rb +1 -1
  79. data/lib/aspera/products/other.rb +2 -2
  80. data/lib/aspera/products/transferd.rb +8 -6
  81. data/lib/aspera/proxy_auto_config.rb +1 -1
  82. data/lib/aspera/rest.rb +29 -22
  83. data/lib/aspera/rest_call_error.rb +1 -1
  84. data/lib/aspera/resumer.rb +1 -1
  85. data/lib/aspera/secret_hider.rb +46 -40
  86. data/lib/aspera/ssh.rb +13 -3
  87. data/lib/aspera/sync/args.schema.yaml +102 -0
  88. data/lib/aspera/sync/conf.schema.yaml +701 -0
  89. data/lib/aspera/sync/database.rb +83 -0
  90. data/lib/aspera/sync/operations.rb +296 -0
  91. data/lib/aspera/temp_file_manager.rb +3 -2
  92. data/lib/aspera/transfer/error.rb +1 -1
  93. data/lib/aspera/transfer/error_info.rb +1 -2
  94. data/lib/aspera/transfer/faux_file.rb +11 -10
  95. data/lib/aspera/transfer/parameters.rb +6 -5
  96. data/lib/aspera/transfer/spec.rb +15 -1
  97. data/lib/aspera/transfer/spec.schema.yaml +316 -293
  98. data/lib/aspera/transfer/spec_doc.rb +34 -16
  99. data/lib/aspera/transfer/uri.rb +5 -5
  100. data/lib/aspera/uri_reader.rb +14 -10
  101. data/lib/aspera/web_auth.rb +2 -2
  102. data/lib/aspera/web_server_simple.rb +2 -2
  103. data.tar.gz.sig +0 -0
  104. metadata +15 -13
  105. metadata.gz.sig +0 -0
  106. data/lib/aspera/transfer/async_conf.schema.yaml +0 -716
  107. data/lib/aspera/transfer/convert.rb +0 -29
  108. data/lib/aspera/transfer/sync.rb +0 -232
  109. data/lib/aspera/transfer/sync_instance.schema.yaml +0 -20
  110. data/lib/aspera/transfer/sync_session.schema.yaml +0 -86
@@ -19,10 +19,10 @@ module Aspera
19
19
  end
20
20
 
21
21
  def create_token
22
- @api.call(
22
+ api.call(
23
23
  operation: 'POST',
24
- subpath: @path_token,
25
- query: @query.merge(scope: @scope), # scope is here because it may change over time (node)
24
+ subpath: path_token,
25
+ query: @query.merge(scope: scope), # scope is here because it may change over time (node)
26
26
  content_type: Rest::MIME_JSON,
27
27
  body: @body,
28
28
  headers: {'Accept' => Rest::MIME_JSON}
@@ -31,8 +31,9 @@ module Aspera
31
31
  # generate secure state to check later
32
32
  random_state = SecureRandom.uuid
33
33
  login_page_url = Rest.build_uri(
34
- "#{@api.base_url}/#{@path_authorize}",
35
- optional_scope_client_id.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state))
34
+ "#{api.base_url}/#{@path_authorize}",
35
+ optional_scope_client_id.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state)
36
+ )
36
37
  # here, we need a human to authorize on a web page
37
38
  Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
38
39
  # start a web server to receive request code
@@ -46,7 +47,8 @@ module Aspera
46
47
  return create_token_call(optional_scope_client_id(add_secret: true).merge(
47
48
  grant_type: 'authorization_code',
48
49
  code: received_params['code'],
49
- redirect_uri: @redirect_uri))
50
+ redirect_uri: @redirect_uri
51
+ ))
50
52
  end
51
53
  end
52
54
  Factory.instance.register_token_creator(Web)
@@ -60,7 +60,7 @@ module Aspera
60
60
  end
61
61
 
62
62
  # Delete persisted items
63
- def garbage_collect(persist_category, max_age_seconds=nil)
63
+ def garbage_collect(persist_category, max_age_seconds = nil)
64
64
  garbage_files = current_files(persist_category)
65
65
  if !max_age_seconds.nil?
66
66
  current_time = Time.now
@@ -75,7 +75,7 @@ module Aspera
75
75
  end
76
76
 
77
77
  def current_files(persist_category)
78
- Dir[File.join(@folder, persist_category + '*' + FILE_SUFFIX)]
78
+ Dir[File.join(@folder, "#{persist_category}*#{FILE_SUFFIX}")]
79
79
  end
80
80
 
81
81
  def current_items(persist_category)
@@ -48,7 +48,8 @@ module Aspera
48
48
  'application/x-xfig' => :image,
49
49
  'font/ttf' => :image,
50
50
  'text/troff' => :image,
51
- 'video/x-mng' => :image}.freeze
51
+ 'video/x-mng' => :image
52
+ }.freeze
52
53
 
53
54
  private_constant :SUPPORTED_MIME_TYPES
54
55
 
@@ -68,7 +69,7 @@ module Aspera
68
69
  return :office if mimetype.start_with?('application/vnd.openxmlformats-officedocument')
69
70
  return :video if mimetype.start_with?('video/')
70
71
  return :image if mimetype.start_with?('image/')
71
- return nil
72
+ return
72
73
  end
73
74
 
74
75
  # @param filepath [String] full path to file
@@ -118,7 +119,7 @@ module Aspera
118
119
  return Environment.secure_capture(exec: 'file', args: ['--mime-type', '--brief', filepath]).strip
119
120
  rescue => e
120
121
  Log.log.error{"error using 'file' command: #{e.message}"}
121
- return nil
122
+ return
122
123
  end
123
124
  end
124
125
  end
@@ -116,7 +116,9 @@ module Aspera
116
116
  '-filter:v', "scale='trunc(iw/2)*2:trunc(ih/2)*2'",
117
117
  '-codec:v', 'libx264',
118
118
  '-r', 30,
119
- '-pix_fmt', 'yuv420p'])
119
+ '-pix_fmt', 'yuv420p'
120
+ ]
121
+ )
120
122
  end
121
123
 
122
124
  # generate n clips starting at offset
@@ -135,7 +137,9 @@ module Aspera
135
137
  '-ss', offset_seconds * 0.1,
136
138
  '-t', @options.clips_length,
137
139
  '-filter:v', "scale=#{@options.video_scale}",
138
- '-codec:a', 'libmp3lame'])
140
+ '-codec:a', 'libmp3lame'
141
+ ]
142
+ )
139
143
  f.puts("file '#{tmp_file_name}'")
140
144
  end
141
145
  end
@@ -144,7 +148,8 @@ module Aspera
144
148
  in_f: file_list_file,
145
149
  in_p: ['-f', 'concat'],
146
150
  out_f: @destination_file_path,
147
- out_p: ['-codec', 'copy'])
151
+ out_p: ['-codec', 'copy']
152
+ )
148
153
  File.delete(file_list_file)
149
154
  end
150
155
 
@@ -174,7 +179,9 @@ module Aspera
174
179
  '-codec:a', 'libmp3lame',
175
180
  '-ac', '2',
176
181
  '-b:a', '128k',
177
- '-movflags', 'faststart'])
182
+ '-movflags', 'faststart'
183
+ ]
184
+ )
178
185
  end
179
186
 
180
187
  def convert_video_to_png_using_fixed
@@ -182,7 +189,8 @@ module Aspera
182
189
  @source_file_path,
183
190
  Utils.video_get_duration(@source_file_path) * @options.thumb_vid_fraction,
184
191
  @options.thumb_vid_scale,
185
- @destination_file_path)
192
+ @destination_file_path
193
+ )
186
194
  end
187
195
 
188
196
  # https://trac.ffmpeg.org/wiki/SponsoringPrograms/GSoC/2015#AnimatedPortableNetworkGraphicsAPNG
@@ -201,19 +209,21 @@ module Aspera
201
209
  '-vf', 'fps=5,scale=120:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse',
202
210
  '-loop', 0,
203
211
  '-f', 'gif'
204
- ])
212
+ ]
213
+ )
205
214
  end
206
215
 
207
216
  def convert_office_to_png
208
- tmp_pdf_file = File.join(this_tmpdir, File.basename(@source_file_path, File.extname(@source_file_path)) + '.pdf')
217
+ tmp_pdf_file = File.join(this_tmpdir, "#{File.basename(@source_file_path, File.extname(@source_file_path))}.pdf")
209
218
  Utils.external_command(:unoconv, [
210
219
  '-f', 'pdf',
211
220
  '-o', tmp_pdf_file,
212
- @source_file_path])
221
+ @source_file_path
222
+ ])
213
223
  convert_pdf_to_png(tmp_pdf_file)
214
224
  end
215
225
 
216
- def convert_pdf_to_png(source_file_path=nil)
226
+ def convert_pdf_to_png(source_file_path = nil)
217
227
  source_file_path ||= @source_file_path
218
228
  Utils.external_command(:magick, [
219
229
  'convert',
@@ -221,7 +231,8 @@ module Aspera
221
231
  '-background', 'white',
222
232
  '-flatten',
223
233
  "#{source_file_path}[0]",
224
- @destination_file_path])
234
+ @destination_file_path
235
+ ])
225
236
  end
226
237
 
227
238
  def convert_image_to_png
@@ -233,7 +244,8 @@ module Aspera
233
244
  '+dither',
234
245
  '-posterize', 40,
235
246
  "#{@source_file_path}[0]",
236
- @destination_file_path])
247
+ @destination_file_path
248
+ ])
237
249
  Utils.external_command(:optipng, [@destination_file_path])
238
250
  end
239
251
 
@@ -253,7 +265,8 @@ module Aspera
253
265
  '-bordercolor', 'white',
254
266
  '-border', 8,
255
267
  '+repage',
256
- @destination_file_path])
268
+ @destination_file_path
269
+ ])
257
270
  end
258
271
  end
259
272
  end
@@ -5,9 +5,11 @@
5
5
  require 'rainbow'
6
6
  require 'io/console'
7
7
  require 'aspera/log'
8
+ require 'aspera/environment'
8
9
  module Aspera
9
10
  module Preview
10
- # Display a picture in the terminal, either using coloured characters or iTerm2
11
+ # Display a picture in the terminal.
12
+ # Either use coloured characters or iTerm2 protocol.
11
13
  class Terminal
12
14
  # Rainbow only supports 8-bit colors
13
15
  # env vars to detect terminal type
@@ -19,13 +21,14 @@ module Aspera
19
21
  DEFAULT_FONT_RATIO = 32.0 / 14.0
20
22
  private_constant :TERM_ENV_VARS, :ITERM_NAMES, :DEFAULT_FONT_RATIO
21
23
  class << self
22
- # @return [String] the image as text, or the iTerm2 escape sequence
23
- # @param blob [String] the image as a binary string
24
- # @param reserve [Integer] number of lines to reserve for other text than the image
25
- # @param text [Boolean] true to display the image as text, false to use iTerm2
26
- # @param double [Boolean] true to use colors on half lines, false to use colors on full lines
27
- # @param font_ratio [Float] ratio = font height / font width
24
+ # @param blob [String] The image as a binary string
25
+ # @param reserve [Integer] Number of lines to reserve for other text than the image
26
+ # @param text [Boolean] `true` to display the image as text, `false` to use iTerm2 if supported
27
+ # @param double [Boolean] `true` to use colors on half lines, `false` to use colors on full lines
28
+ # @param font_ratio [Float] ratio = font height / font width
29
+ # @return [String] The image as text, or the iTerm2 escape sequence
28
30
  def build(blob, reserve: 3, text: false, double: true, font_ratio: DEFAULT_FONT_RATIO)
31
+ return '[Image display requires a terminal]' unless Environment.terminal?
29
32
  return iterm_display_image(blob) if iterm_supported? && !text
30
33
  begin
31
34
  # do not require statically, as the package is optional
@@ -31,7 +31,7 @@ module Aspera
31
31
  end
32
32
 
33
33
  # check that external tools can be executed
34
- def check_tools(skip_types=[])
34
+ def check_tools(skip_types = [])
35
35
  tools_to_check = EXTERNAL_TOOLS.dup
36
36
  tools_to_check.delete(:unoconv) if skip_types.include?(:office)
37
37
  # Check for binaries
@@ -70,7 +70,8 @@ module Aspera
70
70
  '-loglevel', 'error',
71
71
  '-show_entries', 'format=duration',
72
72
  '-print_format', 'default=noprint_wrappers=1:nokey=1', # cspell:disable-line
73
- input_file]).to_f
73
+ input_file
74
+ ]).to_f
74
75
  end
75
76
 
76
77
  def ffmpeg_fmt(temp_folder)
@@ -88,24 +89,25 @@ module Aspera
88
89
  end
89
90
  end
90
91
 
91
- def video_blend_frames(temp_folder, index1, index2)
92
- img1 = get_tmp_num_filepath(temp_folder, index1)
93
- img2 = get_tmp_num_filepath(temp_folder, index2)
94
- count = index2 - index1 - 1
92
+ def video_blend_frames(temp_folder, index_begin, index_end)
93
+ img1 = get_tmp_num_filepath(temp_folder, index_begin)
94
+ img2 = get_tmp_num_filepath(temp_folder, index_end)
95
+ count = index_end - index_begin - 1
95
96
  1.upto(count) do |i|
96
97
  percent = i * 100 / (count + 1)
97
- filename = get_tmp_num_filepath(temp_folder, index1 + i)
98
+ filename = get_tmp_num_filepath(temp_folder, index_begin + i)
98
99
  external_command(:magick, ['composite', '-blend', percent, img2, img1, filename])
99
100
  end
100
101
  end
101
102
 
102
- def video_dump_frame(input_file, offset_seconds, scale, output_file, index=nil)
103
+ def video_dump_frame(input_file, offset_seconds, scale, output_file, index = nil)
103
104
  output_file = get_tmp_num_filepath(output_file, index) unless index.nil?
104
105
  ffmpeg(
105
106
  in_f: input_file,
106
107
  in_p: ['-ss', offset_seconds],
107
108
  out_f: output_file,
108
- out_p: ['-frames:v', 1, '-filter:v', "scale=#{scale}"])
109
+ out_p: ['-frames:v', 1, '-filter:v', "scale=#{scale}"]
110
+ )
109
111
  return output_file
110
112
  end
111
113
  end
@@ -13,7 +13,7 @@ module Aspera
13
13
  class << self
14
14
  # standard folder locations
15
15
  def locations
16
- case Aspera::Environment.os
16
+ case Aspera::Environment.instance.os
17
17
  when Aspera::Environment::OS_WINDOWS then [{
18
18
  app_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Programs', 'Aspera', 'Aspera Connect'),
19
19
  log_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Aspera', 'Aspera Connect', 'var', 'log'),
@@ -11,7 +11,7 @@ module Aspera
11
11
  class << self
12
12
  # standard folder locations
13
13
  def locations
14
- case Aspera::Environment.os
14
+ case Aspera::Environment.instance.os
15
15
  when Aspera::Environment::OS_MACOS then [{
16
16
  app_root: File.join('', 'Applications', 'IBM Aspera.app'),
17
17
  log_root: File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER),
@@ -21,7 +21,7 @@ module Aspera
21
21
  # :log_root O location of log files (Linux uses syslog)
22
22
  # :run_root O only for Connect Client, location of http port file
23
23
  # :sub_bin O subfolder with executables, default : bin
24
- LOCATION_ON_THIS_OS = case Aspera::Environment.os
24
+ LOCATION_ON_THIS_OS = case Aspera::Environment.instance.os
25
25
  when Aspera::Environment::OS_WINDOWS then [{
26
26
  expected: CLI_V3,
27
27
  app_root: File.join('C:', 'Program Files', 'Aspera', 'cli'),
@@ -61,7 +61,7 @@ module Aspera
61
61
  next false unless Dir.exist?(item[:app_root])
62
62
  Log.log.debug{"Found #{item[:expected]}"}
63
63
  sub_bin = item[:sub_bin] || 'bin'
64
- item[:ascp_path] = File.join(item[:app_root], sub_bin, Environment.exe_file('ascp'))
64
+ item[:ascp_path] = File.join(item[:app_root], sub_bin, Environment.instance.exe_file('ascp'))
65
65
  # skip if no ascp
66
66
  next false unless File.exist?(item[:ascp_path])
67
67
  # read info from product info file if present
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/log'
4
+ require 'aspera/assert'
3
5
  module Aspera
4
6
  module Products
5
7
  class Transferd
@@ -19,9 +21,9 @@ module Aspera
19
21
  end
20
22
 
21
23
  # location of SDK files
22
- def sdk_directory=(v)
23
- Log.log.debug{"sdk_directory=#{v}"}
24
- @sdk_dir = v
24
+ def sdk_directory=(folder)
25
+ Log.log.debug{"sdk_directory=#{folder}"}
26
+ @sdk_dir = folder
25
27
  sdk_directory
26
28
  end
27
29
 
@@ -33,9 +35,9 @@ module Aspera
33
35
  end
34
36
 
35
37
  def transferd_path
36
- v1_path = File.join(sdk_directory, Environment.exe_file(V1_DAEMON_NAME))
38
+ v1_path = File.join(sdk_directory, Environment.instance.exe_file(V1_DAEMON_NAME))
37
39
  return v1_path if File.exist?(v1_path)
38
- return File.join(sdk_directory, Environment.exe_file(V2_DAEMON_NAME))
40
+ return File.join(sdk_directory, Environment.instance.exe_file(V2_DAEMON_NAME))
39
41
  end
40
42
 
41
43
  # Well, the port number is only in log file
@@ -51,7 +53,7 @@ module Aspera
51
53
  end
52
54
  end
53
55
  end
54
- raise 'Port not found in daemon logs' if result.nil?
56
+ Aspera.assert(!result.nil?){'Port not found in daemon logs'}
55
57
  Log.log.debug{"Got port #{result} from log"}
56
58
  return result
57
59
  end
@@ -13,7 +13,7 @@ module URI
13
13
  def register_proxy_finder
14
14
  Aspera.assert(block_given?)
15
15
  # overload the method in URI : call user's provided block and fallback to original method
16
- define_method(:find_proxy){ |env_vars=ENV| yield(to_s) || find_proxy_orig(env_vars)}
16
+ define_method(:find_proxy){ |env_vars = ENV| yield(to_s) || find_proxy_orig(env_vars)}
17
17
  end
18
18
  end
19
19
  end
data/lib/aspera/rest.rb CHANGED
@@ -88,11 +88,11 @@ module Aspera
88
88
  # Build URI from URL and parameters and check it is http or https
89
89
  # encode array [] parameters
90
90
  # @param query [Hash,Array]
91
- def build_uri(url, query=nil)
91
+ def build_uri(url, query = nil)
92
92
  uri = URI.parse(url)
93
93
  Aspera.assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
94
94
  return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
95
- Log.log.debug{Log.dump('query', query)}
95
+ Log.dump(:query, query)
96
96
  query_array = []
97
97
  case query
98
98
  when Hash
@@ -194,9 +194,7 @@ module Aspera
194
194
 
195
195
  # create and start keep alive connection on demand
196
196
  def http_session
197
- if @http_session.nil?
198
- @http_session = self.class.start_http_session(@base_url)
199
- end
197
+ @http_session = self.class.start_http_session(@base_url) if @http_session.nil?
200
198
  return @http_session
201
199
  end
202
200
 
@@ -239,7 +237,7 @@ module Aspera
239
237
  )
240
238
  Aspera.assert_type(base_url, String)
241
239
  # base url with no trailing slashes (note: string may be frozen)
242
- @base_url = base_url.gsub(%r{/+$}, '')
240
+ @base_url = base_url.chomp('/')
243
241
  # remove trailing port if it is 443 and scheme is https
244
242
  @base_url = @base_url.gsub(/:443$/, '') if @base_url.start_with?('https://')
245
243
  @base_url = @base_url.gsub(/:80$/, '') if @base_url.start_with?('http://')
@@ -266,7 +264,7 @@ module Aspera
266
264
  if @oauth.nil?
267
265
  Aspera.assert(@auth_params[:type].eql?(:oauth2)){'no OAuth defined'}
268
266
  oauth_parameters = @auth_params.reject{ |k, _v| k.eql?(:type)}
269
- Log.log.debug{Log.dump('oauth parameters', oauth_parameters)}
267
+ Log.dump(:oauth_parameters, oauth_parameters)
270
268
  @oauth = OAuth::Factory.instance.create(**oauth_parameters)
271
269
  end
272
270
  return @oauth
@@ -293,8 +291,8 @@ module Aspera
293
291
  )
294
292
  subpath = subpath.to_s if subpath.is_a?(Symbol)
295
293
  subpath = '' if subpath.nil?
296
- Log.log.debug{"#{operation} [#{subpath}]".red.bold.bg_green}
297
- Log.log.debug{Log.dump(:body, body)}
294
+ Log.log.debug{"call #{operation} [#{subpath}]".red.bold.bg_green}
295
+ Log.dump(:body, body)
298
296
  Aspera.assert_type(subpath, String)
299
297
  if headers.nil?
300
298
  headers = @headers.clone
@@ -352,7 +350,7 @@ module Aspera
352
350
  end
353
351
  # :type = :basic
354
352
  req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
355
- Log.log.trace1{Log.dump(:req_body, req.body)}
353
+ Log.dump(:req_body, req.body, level: :trace1)
356
354
  # we try the call, and will retry on some error types
357
355
  error_tries ||= 1 + RestParameters.instance.retry_max
358
356
  # initialize with number of initial retries allowed, nil gives zero
@@ -375,9 +373,7 @@ module Aspera
375
373
  # override user's path to path in header
376
374
  if !response['Content-Disposition'].nil?
377
375
  disposition = self.class.parse_header(response['Content-Disposition'])
378
- if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
379
- target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename])
380
- end
376
+ target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename]) if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
381
377
  end
382
378
  # download with temp filename
383
379
  target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
@@ -395,7 +391,8 @@ module Aspera
395
391
  RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
396
392
  end
397
393
  end
398
- RestParameters.instance.progress_bar&.event(:end, session_id: session_id)
394
+ RestParameters.instance.progress_bar&.event(:session_end, session_id: session_id)
395
+ RestParameters.instance.progress_bar&.event(:end)
399
396
  # rename at the end
400
397
  File.rename(target_file_tmp, target_file)
401
398
  file_saved = true
@@ -408,7 +405,7 @@ module Aspera
408
405
  case result_mime
409
406
  when *JSON_DECODE
410
407
  result[:data] = JSON.parse(result[:http].body) rescue result[:http].body
411
- Log.log.debug{Log.dump('result_data', result[:data])}
408
+ Log.dump(:result_data, result[:data])
412
409
  else # when MIME_TEXT
413
410
  result[:data] = result[:http].body
414
411
  end
@@ -447,7 +444,7 @@ module Aspera
447
444
  if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
448
445
  tries_remain_redirect -= 1
449
446
  current_uri = URI.parse(@base_url)
450
- new_url = e.response['location']
447
+ new_url = e.response['Location']
451
448
  # special case: relative redirect
452
449
  if URI.parse(new_url).host.nil?
453
450
  # we don't manage relative redirects with non-absolute path
@@ -455,9 +452,19 @@ module Aspera
455
452
  new_url = "#{current_uri.scheme}://#{current_uri.host}#{new_url}"
456
453
  end
457
454
  # forwards the request to the new location
458
- return self.class.new(base_url: new_url, redirect_max: tries_remain_redirect).call(
459
- operation: operation, query: query, body: body, content_type: content_type,
460
- save_to_file: save_to_file, return_error: return_error, headers: headers)
455
+ return self.class.new(
456
+ base_url: new_url,
457
+ redirect_max: tries_remain_redirect
458
+ ).call(
459
+ operation: operation,
460
+ subpath: new_url.end_with?('/') ? '/' : nil,
461
+ query: query,
462
+ body: body,
463
+ content_type: content_type,
464
+ save_to_file: save_to_file,
465
+ return_error: return_error,
466
+ headers: headers
467
+ )
461
468
  end
462
469
  # raise exception if could not retry and not return error in result
463
470
  raise e unless return_error
@@ -475,7 +482,7 @@ module Aspera
475
482
  return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
476
483
  end
477
484
 
478
- def read(subpath, query=nil)
485
+ def read(subpath, query = nil)
479
486
  return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query)[:data]
480
487
  end
481
488
 
@@ -483,7 +490,7 @@ module Aspera
483
490
  return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
484
491
  end
485
492
 
486
- def delete(subpath, params=nil)
493
+ def delete(subpath, params = nil)
487
494
  return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params)[:data]
488
495
  end
489
496
 
@@ -513,7 +520,7 @@ module Aspera
513
520
  name_matches = matching_items.select{ |i| i['name'].casecmp?(search_name)}
514
521
  case name_matches.length
515
522
  when 1 then return name_matches.first
516
- when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.) # rubocop:disable Layout/LineLength
523
+ when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
517
524
  else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{ |i| i['name']}}"
518
525
  end
519
526
  end
@@ -8,7 +8,7 @@ module Aspera
8
8
  # @param req HTTP Request object
9
9
  # @param resp HTTP Response object
10
10
  # @param msg Error message
11
- def initialize(msg, req=nil, resp=nil)
11
+ def initialize(msg, req = nil, resp = nil)
12
12
  @request = req
13
13
  @response = resp
14
14
  super(msg)
@@ -16,7 +16,7 @@ module Aspera
16
16
  }.freeze
17
17
 
18
18
  # @param params see DEFAULTS
19
- def initialize(params=nil)
19
+ def initialize(params = nil)
20
20
  @parameters = DEFAULTS.dup
21
21
  if !params.nil?
22
22
  Aspera.assert_type(params, Hash)
@@ -2,10 +2,13 @@
2
2
 
3
3
  # cspell:ignore FILEPASS
4
4
  require 'logger'
5
+ require 'singleton'
5
6
 
6
7
  module Aspera
7
8
  # remove secret from logs and output
8
9
  class SecretHider
10
+ include Singleton
11
+
9
12
  # configurable:
10
13
  ADDITIONAL_KEYS_TO_HIDE = []
11
14
  # display string for hidden secrets
@@ -34,56 +37,59 @@ module Aspera
34
37
  /(?<begin>(?:#{HTTP_SECRETS.join('|')}): )[^\\]+(?<end>\\)/i
35
38
  ].freeze
36
39
  private_constant :HIDDEN_PASSWORD, :ASCP_ENV_SECRETS, :KEY_SECRETS, :HTTP_SECRETS, :ALL_SECRETS, :KEY_FALSE_POSITIVES, :REGEX_LOG_REPLACES
37
- @log_secrets = false
38
- class << self
39
- attr_accessor :log_secrets
40
+ attr_accessor :log_secrets
40
41
 
41
- # @return new log formatter that hides secrets
42
- def log_formatter(original_formatter)
43
- original_formatter ||= Logger::Formatter.new
44
- # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
45
- return lambda do |severity, date_time, program_name, msg|
46
- if msg.is_a?(String) && !@log_secrets
47
- REGEX_LOG_REPLACES.each do |reg_ex|
48
- msg = msg.gsub(reg_ex){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
49
- end
42
+ # @return new log formatter that hides secrets
43
+ def log_formatter(original_formatter)
44
+ original_formatter ||= Logger::Formatter.new
45
+ # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
46
+ return lambda do |severity, date_time, program_name, msg|
47
+ if msg.is_a?(String) && !@log_secrets
48
+ REGEX_LOG_REPLACES.each do |reg_ex|
49
+ msg = msg.gsub(reg_ex){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
50
50
  end
51
- original_formatter.call(severity, date_time, program_name, msg)
52
51
  end
52
+ original_formatter.call(severity, date_time, program_name, msg)
53
53
  end
54
+ end
54
55
 
55
- def hide_secrets_in_string(value)
56
- return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
57
- end
56
+ def hide_secrets_in_string(value)
57
+ return value.gsub(REGEX_LOG_REPLACES.first){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
58
+ end
58
59
 
59
- # @return true if the key denotes a secret
60
- def secret?(keyword, value)
61
- keyword = keyword.to_s if keyword.is_a?(Symbol)
62
- # only Strings can be secrets, not booleans, or hash, arrays
63
- return false unless keyword.is_a?(String) && value.is_a?(String)
64
- # those are not secrets
65
- return false if KEY_FALSE_POSITIVES.any?{ |f| f.match?(keyword)}
66
- return true if ADDITIONAL_KEYS_TO_HIDE.include?(keyword)
67
- # check if keyword (name) contains an element that designate it as a secret
68
- ALL_SECRETS.any?{ |kw| keyword.include?(kw)}
69
- end
60
+ # @return true if the key denotes a secret
61
+ def secret?(keyword, value)
62
+ keyword = keyword.to_s if keyword.is_a?(Symbol)
63
+ # only Strings can be secrets, not booleans, or hash, arrays
64
+ return false unless keyword.is_a?(String) && value.is_a?(String)
65
+ # those are not secrets
66
+ return false if KEY_FALSE_POSITIVES.any?{ |f| f.match?(keyword)}
67
+ return true if ADDITIONAL_KEYS_TO_HIDE.include?(keyword)
68
+ # check if keyword (name) contains an element that designate it as a secret
69
+ ALL_SECRETS.any?{ |kw| keyword.include?(kw)}
70
+ end
70
71
 
71
- # Hides recursively secrets in Hash or Array of Hash
72
- def deep_remove_secret(obj)
73
- case obj
74
- when Array
75
- obj.each{ |i| deep_remove_secret(i)}
76
- when Hash
77
- obj.each do |k, v|
78
- if secret?(k, v)
79
- obj[k] = HIDDEN_PASSWORD
80
- elsif obj[k].is_a?(Hash)
81
- deep_remove_secret(obj[k])
82
- end
72
+ # Hides recursively secrets in Hash or Array of Hash
73
+ def deep_remove_secret(obj)
74
+ case obj
75
+ when Array
76
+ obj.each{ |i| deep_remove_secret(i)}
77
+ when Hash
78
+ obj.each do |k, v|
79
+ if secret?(k, v)
80
+ obj[k] = HIDDEN_PASSWORD
81
+ elsif obj[k].is_a?(Hash)
82
+ deep_remove_secret(obj[k])
83
83
  end
84
84
  end
85
- return obj
86
85
  end
86
+ return obj
87
+ end
88
+
89
+ private
90
+
91
+ def initialize
92
+ @log_secrets = false
87
93
  end
88
94
  end
89
95
  end