aspera-cli 4.12.0 → 4.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +45 -5
  4. data/CONTRIBUTING.md +113 -22
  5. data/README.md +1289 -754
  6. data/bin/ascli +3 -3
  7. data/examples/dascli +1 -1
  8. data/examples/rubyc +24 -0
  9. data/lib/aspera/aoc.rb +63 -74
  10. data/lib/aspera/ascmd.rb +5 -3
  11. data/lib/aspera/cli/basic_auth_plugin.rb +6 -6
  12. data/lib/aspera/cli/extended_value.rb +24 -37
  13. data/lib/aspera/cli/formatter.rb +23 -25
  14. data/lib/aspera/cli/info.rb +2 -4
  15. data/lib/aspera/cli/main.rb +27 -27
  16. data/lib/aspera/cli/manager.rb +143 -120
  17. data/lib/aspera/cli/plugin.rb +88 -43
  18. data/lib/aspera/cli/plugins/alee.rb +2 -2
  19. data/lib/aspera/cli/plugins/aoc.rb +235 -104
  20. data/lib/aspera/cli/plugins/ats.rb +16 -18
  21. data/lib/aspera/cli/plugins/bss.rb +3 -3
  22. data/lib/aspera/cli/plugins/config.rb +190 -373
  23. data/lib/aspera/cli/plugins/console.rb +4 -6
  24. data/lib/aspera/cli/plugins/cos.rb +12 -13
  25. data/lib/aspera/cli/plugins/faspex.rb +21 -21
  26. data/lib/aspera/cli/plugins/faspex5.rb +399 -150
  27. data/lib/aspera/cli/plugins/node.rb +260 -174
  28. data/lib/aspera/cli/plugins/orchestrator.rb +15 -18
  29. data/lib/aspera/cli/plugins/preview.rb +40 -62
  30. data/lib/aspera/cli/plugins/server.rb +33 -16
  31. data/lib/aspera/cli/plugins/shares.rb +24 -33
  32. data/lib/aspera/cli/plugins/sync.rb +6 -6
  33. data/lib/aspera/cli/transfer_agent.rb +47 -30
  34. data/lib/aspera/cli/version.rb +2 -1
  35. data/lib/aspera/colors.rb +9 -7
  36. data/lib/aspera/command_line_builder.rb +2 -1
  37. data/lib/aspera/cos_node.rb +1 -1
  38. data/lib/aspera/data/6 +0 -0
  39. data/lib/aspera/environment.rb +7 -3
  40. data/lib/aspera/fasp/agent_connect.rb +6 -1
  41. data/lib/aspera/fasp/agent_direct.rb +17 -17
  42. data/lib/aspera/fasp/agent_httpgw.rb +138 -60
  43. data/lib/aspera/fasp/agent_node.rb +14 -4
  44. data/lib/aspera/fasp/agent_trsdk.rb +2 -0
  45. data/lib/aspera/fasp/error_info.rb +2 -0
  46. data/lib/aspera/fasp/installation.rb +19 -19
  47. data/lib/aspera/fasp/parameters.rb +29 -20
  48. data/lib/aspera/fasp/parameters.yaml +5 -2
  49. data/lib/aspera/fasp/resume_policy.rb +3 -3
  50. data/lib/aspera/fasp/transfer_spec.rb +8 -5
  51. data/lib/aspera/fasp/uri.rb +23 -21
  52. data/lib/aspera/faspex_gw.rb +1 -0
  53. data/lib/aspera/faspex_postproc.rb +3 -3
  54. data/lib/aspera/hash_ext.rb +12 -2
  55. data/lib/aspera/keychain/macos_security.rb +13 -13
  56. data/lib/aspera/log.rb +1 -0
  57. data/lib/aspera/node.rb +73 -84
  58. data/lib/aspera/oauth.rb +4 -3
  59. data/lib/aspera/persistency_action_once.rb +1 -1
  60. data/lib/aspera/preview/file_types.rb +8 -6
  61. data/lib/aspera/preview/generator.rb +23 -11
  62. data/lib/aspera/preview/options.rb +3 -2
  63. data/lib/aspera/preview/terminal.rb +80 -0
  64. data/lib/aspera/preview/utils.rb +11 -11
  65. data/lib/aspera/proxy_auto_config.js +2 -2
  66. data/lib/aspera/rest.rb +42 -4
  67. data/lib/aspera/rest_call_error.rb +3 -1
  68. data/lib/aspera/secret_hider.rb +10 -5
  69. data/lib/aspera/ssh.rb +1 -1
  70. data/lib/aspera/sync.rb +41 -33
  71. data/lib/aspera/web_server_simple.rb +22 -18
  72. data.tar.gz.sig +0 -0
  73. metadata +40 -48
  74. metadata.gz.sig +0 -0
  75. data/docs/test_env.conf +0 -179
  76. data/examples/aoc.rb +0 -30
  77. data/examples/faspex4.rb +0 -94
  78. data/examples/node.rb +0 -96
  79. data/examples/server.rb +0 -93
  80. data/lib/aspera/data/7 +0 -0
@@ -1,5 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # ffmpeg options:
4
+ # spellchecker:ignore pauseframes libx264 trunc bufsize muxer apng libmp3lame maxrate posterize movflags faststart
5
+ # spellchecker:ignore palettegen paletteuse pointsize bordercolor repage lanczos
6
+
3
7
  require 'open3'
4
8
  require 'aspera/preview/options'
5
9
  require 'aspera/preview/utils'
@@ -12,6 +16,8 @@ module Aspera
12
16
  # values for preview_format : output format
13
17
  PREVIEW_FORMATS = %i[png mp4].freeze
14
18
 
19
+ FFMPEG_OPTIONS_LIST = %w[in out].freeze
20
+
15
21
  # CLI needs to know conversion type to know if need skip it
16
22
  attr_reader :conversion_type
17
23
 
@@ -22,7 +28,6 @@ module Aspera
22
28
  # supported preview type is one of Preview::PREVIEW_FORMATS
23
29
  # the resulting preview file type is taken from destination file extension.
24
30
  # conversion methods are provided by private methods: convert_<conversion_type>_to_<preview_format>
25
- # (combi = combination of source file type and destination format)
26
31
  # -> conversion_type is one of FileTypes::CONVERSION_TYPES
27
32
  # -> preview_format is one of Generator::PREVIEW_FORMATS
28
33
  # the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
@@ -70,10 +75,10 @@ module Aspera
70
75
  # check that generated size does not exceed maximum
71
76
  result_size = File.size(@destination_file_path)
72
77
  if result_size > @options.max_size
73
- Log.log.warn{"preview size exceeds maximum #{result_size} > #{@options.max_size}"}
78
+ Log.log.warn{"preview size exceeds maximum allowed #{result_size} > #{@options.max_size}"}
74
79
  end
75
80
  rescue StandardError => e
76
- Log.log.error{"Ignoging: #{e.message}"}
81
+ Log.log.error{"Ignoring: #{e.message}"}
77
82
  Log.log.debug(e.backtrace.join("\n").red)
78
83
  FileUtils.cp(File.expand_path(@preview_format_symb.eql?(:mp4) ? 'video_error.png' : 'image_error.png', File.dirname(__FILE__)), @destination_file_path)
79
84
  ensure
@@ -102,11 +107,11 @@ module Aspera
102
107
  def convert_video_to_mp4_using_blend
103
108
  p_duration = Utils.video_get_duration(@source_file_path)
104
109
  p_start_offset = @options.video_start_sec.to_i
105
- p_keyframecount = @options.blend_keyframes.to_i
110
+ p_key_frame_count = @options.blend_keyframes.to_i
106
111
  last_keyframe = nil
107
112
  current_index = 1
108
- 1.upto(p_keyframecount) do |i|
109
- offset_seconds = get_offset(p_duration, p_start_offset, p_keyframecount, i)
113
+ 1.upto(p_key_frame_count) do |i|
114
+ offset_seconds = get_offset(p_duration, p_start_offset, p_key_frame_count, i)
110
115
  Utils.video_dump_frame(@source_file_path, offset_seconds, @options.video_scale, this_tmpdir, current_index)
111
116
  Utils.video_dupe_frame(this_tmpdir, current_index, @options.blend_pauseframes)
112
117
  Utils.video_blend_frames(this_tmpdir, last_keyframe, current_index) unless last_keyframe.nil?
@@ -133,17 +138,17 @@ module Aspera
133
138
  File.open(filelist, 'w+') do |f|
134
139
  1.upto(@options.clips_count.to_i) do |i|
135
140
  offset_seconds = get_offset(p_duration, @options.video_start_sec.to_i, @options.clips_count.to_i, i)
136
- tmpfilename = format('clip%04d.mp4', i)
141
+ tmp_file_name = format('clip%04d.mp4', i)
137
142
  Utils.ffmpeg(
138
143
  in_f: @source_file_path,
139
144
  in_p: ['-ss', offset_seconds * 0.9],
140
- out_f: File.join(this_tmpdir, tmpfilename),
145
+ out_f: File.join(this_tmpdir, tmp_file_name),
141
146
  out_p: [
142
147
  '-ss', offset_seconds * 0.1,
143
148
  '-t', @options.clips_length,
144
149
  '-filter:v', "scale=#{@options.video_scale}",
145
150
  '-codec:a', 'libmp3lame'])
146
- f.puts("file '#{tmpfilename}'")
151
+ f.puts("file '#{tmp_file_name}'")
147
152
  end
148
153
  end
149
154
  # concat clips
@@ -154,12 +159,19 @@ module Aspera
154
159
  out_p: ['-codec', 'copy'])
155
160
  end
156
161
 
157
- # do a simple reencoding
162
+ # do a simple re-encoding
158
163
  def convert_video_to_mp4_using_reencode
164
+ options = @options.reencode_ffmpeg
165
+ raise 'reencode_ffmpeg must be a Hash' unless options.is_a?(Hash)
166
+ options.each do |k, v|
167
+ raise "Key not supported: #{k}. Use keys: #{FFMPEG_OPTIONS_LIST.join(',')}" unless FFMPEG_OPTIONS_LIST.include?(k)
168
+ raise "Value for #{k} must be Array" unless v.is_a?(Array)
169
+ end
159
170
  Utils.ffmpeg(
160
171
  in_f: @source_file_path,
172
+ in_p: options['in'] || ['-ss', @options.video_start_sec.to_i * 0.9],
161
173
  out_f: @destination_file_path,
162
- out_p: [
174
+ out_p: options['out'] || [
163
175
  '-t', '60',
164
176
  '-codec:v', 'libx264',
165
177
  '-profile:v', 'high',
@@ -20,8 +20,9 @@ module Aspera
20
20
  { name: :thumb_text_font, default: 'Courier', description: 'png: plaintext: font to render text with imagemagick convert (identify -list font)'},
21
21
  { name: :video_conversion, default: :reencode, description: 'mp4: method for preview generation', values: VIDEO_CONVERSION_METHODS },
22
22
  { name: :video_png_conv, default: :fixed, description: 'mp4: method for thumbnail generation', values: VIDEO_THUMBNAIL_METHODS },
23
- { name: :video_start_sec, default: 10, description: 'mp4: start offset (seconds) of video preview' },
24
- { name: :video_scale, default: "'min(iw,360)':-2", description: 'mp4: video scale (ffmpeg)' },
23
+ { name: :video_scale, default: "'min(iw,360)':-2", description: 'mp4: all: video scale (ffmpeg)' },
24
+ { name: :video_start_sec, default: 10, description: 'mp4: all: start offset (seconds) of video preview' },
25
+ { name: :reencode_ffmpeg, default: {}, description: 'mp4: reencode: options to ffmpeg' },
25
26
  { name: :blend_keyframes, default: 30, description: 'mp4: blend: # key frames' },
26
27
  { name: :blend_pauseframes, default: 3, description: 'mp4: blend: # pause frames' },
27
28
  { name: :blend_transframes, default: 5, description: 'mp4: blend: # transition blend frames' },
@@ -0,0 +1,80 @@
1
+ # frozen_string_literal: true
2
+
3
+ # cspell:words Magick MAGICKCORE ITERM mintty winsize termcap
4
+
5
+ require 'rmagick' # https://rmagick.github.io/index.html
6
+ require 'rainbow'
7
+ require 'io/console'
8
+ module Aspera
9
+ module Preview
10
+ # Generates a string that can display an image in a terminal
11
+ class Terminal
12
+ # quantum depth is 8 or 16: convert xc: -format "%q" info:
13
+ # Rainbow only supports 8-bit colors
14
+ SHIFT_FOR_8_BIT = Magick::MAGICKCORE_QUANTUM_DEPTH - 8
15
+ ITERM_NAMES = %w[iTerm WezTerm mintty].freeze
16
+ TERM_ENV_VARS = %w[TERM_PROGRAM LC_TERMINAL].freeze
17
+ private_constant :SHIFT_FOR_8_BIT, :ITERM_NAMES, :TERM_ENV_VARS
18
+ class << self
19
+ def build(blob, reserved_lines: 0, double_precision: true)
20
+ return iterm_display_image(blob) if iterm_supported?
21
+ image = Magick::ImageList.new.from_blob(blob)
22
+ (term_rows, term_columns) = IO.console.winsize
23
+ term_rows -= reserved_lines
24
+ # compute scaling to fit terminal
25
+ fit_term_ratio = [term_rows / image.rows.to_f, term_columns / image.columns.to_f].min
26
+ # TODO: retrieve terminal font ratio using some termcap ?
27
+ font_ratio = 1.7
28
+ height_ratio = double_precision ? 2.0 : 1.0
29
+ image = image.scale((image.columns * fit_term_ratio * font_ratio).to_i, (image.rows * fit_term_ratio * height_ratio).to_i)
30
+ # get all pixel colors, adjusted for Rainbow
31
+ pixel_colors = []
32
+ image.each_pixel do |pixel, col, row|
33
+ pixel_rgb = [pixel.red, pixel.green, pixel.blue]
34
+ pixel_rgb = pixel_rgb.map { |color| color >> SHIFT_FOR_8_BIT } unless SHIFT_FOR_8_BIT.eql?(0)
35
+ # init 2-dim array
36
+ pixel_colors[row] ||= []
37
+ pixel_colors[row][col] = pixel_rgb
38
+ end
39
+ # now generate text
40
+ text_pixels = []
41
+ pixel_colors.each_with_index do |row_data, row|
42
+ next if double_precision && row.odd?
43
+ row_data.each_with_index do |pixel_rgb, col|
44
+ text_pixels.push("\n") if col.eql?(0) && !row.eql?(0)
45
+ if double_precision
46
+ text_pixels.push(Rainbow('▄').background(pixel_rgb).foreground(pixel_colors[row + 1][col]))
47
+ else
48
+ text_pixels.push(Rainbow(' ').background(pixel_rgb))
49
+ end
50
+ end
51
+ end
52
+ return text_pixels.join
53
+ end
54
+
55
+ # display image in iTerm2
56
+ def iterm_display_image(blob)
57
+ # image = Magick::ImageList.new.from_blob(blob)
58
+ arguments = {
59
+ inline: 1,
60
+ preserveAspectRatio: 1,
61
+ size: blob.length
62
+ # width: image.columns,
63
+ # height: image.rows
64
+ }.map { |k, v| "#{k}=#{v}" }.join(';')
65
+ # \a is BEL, \e is ESC : https://github.com/ruby/ruby/blob/master/doc/syntax/literals.rdoc#label-Strings
66
+ # https://iterm2.com/documentation-images.html
67
+ return "\e]1337;File=#{arguments}:#{Base64.encode64(blob)}\a"
68
+ end
69
+
70
+ # @return [Boolean] true if the terminal supports iTerm2 image display
71
+ def iterm_supported?
72
+ TERM_ENV_VARS.each do |env_var|
73
+ return true if ITERM_NAMES.any? { |term| ENV[env_var]&.include?(term) }
74
+ end
75
+ false
76
+ end
77
+ end # class << self
78
+ end # class Terminal
79
+ end # module Preview
80
+ end # module Aspera
@@ -14,12 +14,12 @@ module Aspera
14
14
  # shell exit code when command is not found
15
15
  BASH_EXIT_NOT_FOUND = 127
16
16
  # external binaries used
17
- EXPERNAL_TOOLS = %i[ffmpeg ffprobe convert composite optipng unoconv].freeze
18
- TMPFMT = 'img%04d.jpg'
19
- private_constant :BASH_SPECIAL_CHARACTERS, :BASH_EXIT_NOT_FOUND, :EXPERNAL_TOOLS, :TMPFMT
17
+ EXTERNAL_TOOLS = %i[ffmpeg ffprobe convert composite optipng unoconv].freeze
18
+ TEMP_FORMAT = 'img%04d.jpg'
19
+ private_constant :BASH_SPECIAL_CHARACTERS, :BASH_EXIT_NOT_FOUND, :EXTERNAL_TOOLS, :TEMP_FORMAT
20
20
 
21
21
  class << self
22
- # returns string with single quotes suitable for bash if there is any bash metacharacter
22
+ # returns string with single quotes suitable for bash if there is any bash meta-character
23
23
  def shell_quote(argument)
24
24
  return argument unless argument.chars.any?{|c|BASH_SPECIAL_CHARACTERS.include?(c)}
25
25
  return "'" + argument.gsub(/'/){|_s| "'\"'\"'"} + "'"
@@ -27,7 +27,7 @@ module Aspera
27
27
 
28
28
  # check that external tools can be executed
29
29
  def check_tools(skip_types=[])
30
- tools_to_check = EXPERNAL_TOOLS.dup
30
+ tools_to_check = EXTERNAL_TOOLS.dup
31
31
  tools_to_check.delete(:unoconv) if skip_types.include?(:office)
32
32
  # Check for binaries
33
33
  tools_to_check.each do |command_symb|
@@ -39,7 +39,7 @@ module Aspera
39
39
  # one could use "system", but we would need to redirect stdout/err
40
40
  # @return true if su
41
41
  def external_command(command_symb, command_args)
42
- raise "unexpected command #{command_symb}" unless EXPERNAL_TOOLS.include?(command_symb)
42
+ raise "unexpected command #{command_symb}" unless EXTERNAL_TOOLS.include?(command_symb)
43
43
  # build command line, and quote special characters
44
44
  command = command_args.clone.unshift(command_symb).map{|i| shell_quote(i.to_s)}.join(' ')
45
45
  Log.log.debug{"cmd=#{command}".blue}
@@ -55,7 +55,7 @@ module Aspera
55
55
  raise "Error: #{command_symb} is not in the PATH"
56
56
  end
57
57
  unless exit_status.success?
58
- Log.log.error{"commandline: #{command}"}
58
+ Log.log.error{"command line: #{command}"}
59
59
  Log.log.error{"Error code: #{exit_status}"}
60
60
  Log.log.error{"stdout: #{stdout}"}
61
61
  Log.log.error{"stderr: #{stderr}"}
@@ -69,7 +69,7 @@ module Aspera
69
69
  # input_file,input_args,output_file,output_args
70
70
  a[:gl_p] ||= [
71
71
  '-y', # overwrite output without asking
72
- '-loglevel', 'error' # show only errors and up]
72
+ '-loglevel', 'error' # show only errors and up
73
73
  ]
74
74
  a[:in_p] ||= []
75
75
  a[:out_p] ||= []
@@ -82,17 +82,17 @@ module Aspera
82
82
  result = external_command(:ffprobe, [
83
83
  '-loglevel', 'error',
84
84
  '-show_entries', 'format=duration',
85
- '-print_format', 'default=noprint_wrappers=1:nokey=1',
85
+ '-print_format', 'default=noprint_wrappers=1:nokey=1', # cspell:disable-line
86
86
  input_file])
87
87
  return result[:stdout].to_f
88
88
  end
89
89
 
90
90
  def ffmpeg_fmt(temp_folder)
91
- return File.join(temp_folder, TMPFMT)
91
+ return File.join(temp_folder, TEMP_FORMAT)
92
92
  end
93
93
 
94
94
  def get_tmp_num_filepath(temp_folder, file_number)
95
- return File.join(temp_folder, format(TMPFMT, file_number))
95
+ return File.join(temp_folder, format(TEMP_FORMAT, file_number))
96
96
  end
97
97
 
98
98
  def video_dupe_frame(temp_folder, index, count)
@@ -70,7 +70,7 @@ function shExpMatch(str, shell_expr) {
70
70
  }
71
71
  function weekdayRange(wd1, wd2, gmt) {
72
72
  var today = new Date();
73
- var days = 'SUNMONTUEWEDTHUFRISAT';
73
+ var days = 'SUNMONTUEWEDTHUFRISAT'; // cspell: disable-line
74
74
  wd1 = wd1.toUpperCase();
75
75
  if (wd2 == undefined)
76
76
  wd2 = wd1;
@@ -138,7 +138,7 @@ function dateRange() {
138
138
  else
139
139
  return false;
140
140
  } else if (typeof arg == 'string') {
141
- var months = 'JANFEBMARAPRMAYJUNJULAUGSEPOCTNOVDEC';
141
+ var months = 'JANFEBMARAPRMAYJUNJULAUGSEPOCTNOVDEC'; // cspell: disable-line
142
142
  arg = arg.toUpperCase();
143
143
  arg = months.indexOf(arg);
144
144
  if (arg == -1)
data/lib/aspera/rest.rb CHANGED
@@ -38,6 +38,11 @@ module Aspera
38
38
 
39
39
  ARRAY_PARAMS = '[]'
40
40
 
41
+ private_constant :ARRAY_PARAMS
42
+
43
+ # error message when entity not found
44
+ ENTITY_NOT_FOUND = 'No such'
45
+
41
46
  class << self
42
47
  # define accessors
43
48
  @@global.each_key do |p|
@@ -50,6 +55,12 @@ module Aspera
50
55
 
51
56
  def basic_creds(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
52
57
 
58
+ # used to build a parameter list prefixed with "[]"
59
+ # @param values [Array] list of values
60
+ def array_params(values)
61
+ return [ARRAY_PARAMS].concat(values)
62
+ end
63
+
53
64
  # build URI from URL and parameters and check it is http or https
54
65
  def build_uri(url, params=nil)
55
66
  uri = URI.parse(url)
@@ -224,8 +235,9 @@ module Aspera
224
235
  begin
225
236
  # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
226
237
  oauth_tries ||= 2
227
- tries_remain_redirect ||= call_data[:redirect_max].nil? ? 0 : call_data[:redirect_max].to_i
228
- Log.log.debug('send request')
238
+ # initialize with number of initial retries allowed, nil gives zero
239
+ tries_remain_redirect = call_data[:redirect_max].to_i if tries_remain_redirect.nil?
240
+ Log.log.debug("send request (retries=#{tries_remain_redirect})")
229
241
  # make http request (pipelined)
230
242
  http_session.request(req) do |response|
231
243
  result[:http] = response
@@ -286,8 +298,8 @@ module Aspera
286
298
  Log.log.debug{"using new token=#{call_data[:headers]['Authorization']}"}
287
299
  retry unless (oauth_tries -= 1).zero?
288
300
  end # if oauth
289
- # moved ?
290
- if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
301
+ # redirect ? (any code beginning with 3)
302
+ if tries_remain_redirect.positive? && e.response.is_a?(Net::HTTPRedirection)
291
303
  tries_remain_redirect -= 1
292
304
  current_uri = URI.parse(call_data[:base_url])
293
305
  new_url = e.response['location']
@@ -336,5 +348,31 @@ module Aspera
336
348
  def cancel(subpath)
337
349
  return call({operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'}})
338
350
  end
351
+
352
+ # Query by name and returns a single result, else it throws an exception (no or multiple results)
353
+ # @param subpath path of entity in API
354
+ # @param search_name name of searched entity
355
+ # @param options additional search options
356
+ def lookup_by_name(subpath, search_name, options={})
357
+ # returns entities whose name contains value (case insensitive)
358
+ matching_items = read(subpath, options.merge({'q' => CGI.escape(search_name)}))[:data]
359
+ # API style: {totalcount:, ...} cspell: disable-line
360
+ # TODO: not generic enough ? move somewhere ? inheritance ?
361
+ matching_items = matching_items[subpath] if matching_items.is_a?(Hash)
362
+ raise "Internal error: expecting array, have #{matching_items.class}" unless matching_items.is_a?(Array)
363
+ case matching_items.length
364
+ when 1 then return matching_items.first
365
+ when 0 then raise %Q{#{ENTITY_NOT_FOUND} #{subpath}: "#{search_name}"}
366
+ else
367
+ # multiple case insensitive partial matches, try case insensitive full match
368
+ # (anyway AoC does not allow creation of 2 entities with same case insensitive name)
369
+ name_matches = matching_items.select{|i|i['name'].casecmp?(search_name)}
370
+ case name_matches.length
371
+ when 1 then return name_matches.first
372
+ when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{|i|i['name']}} but no case insensitive full match. Please be more specific or give exact name.) # rubocop:disable Layout/LineLength
373
+ else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{|i|i['name']}}"
374
+ end
375
+ end
376
+ end
339
377
  end
340
378
  end # module Aspera
@@ -5,7 +5,9 @@ module Aspera
5
5
  class RestCallError < StandardError
6
6
  attr_accessor :request, :response
7
7
 
8
- # @param http response
8
+ # @param req HTTP Request object
9
+ # @param resp HTTP Response object
10
+ # @param msg Error message
9
11
  def initialize(req, resp, msg)
10
12
  @request = req
11
13
  @response = resp
@@ -12,6 +12,7 @@ module Aspera
12
12
  # keys in hash that contain secrets
13
13
  KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
14
14
  ALL_SECRETS = [ASCP_ENV_SECRETS, KEY_SECRETS].flatten.freeze
15
+ FALSE_POSITIVES = [/^access_key$/].freeze
15
16
  # regex that define named captures :begin and :end
16
17
  REGEX_LOG_REPLACES = [
17
18
  # CLI manager get/set options
@@ -35,20 +36,24 @@ module Aspera
35
36
  def log_formatter(original_formatter)
36
37
  original_formatter ||= Logger::Formatter.new
37
38
  # NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
38
- return lambda do |severity, datetime, progname, msg|
39
+ return lambda do |severity, date_time, program_name, msg|
39
40
  if msg.is_a?(String) && !@log_secrets
40
- REGEX_LOG_REPLACES.each do |regx|
41
- msg = msg.gsub(regx){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
41
+ REGEX_LOG_REPLACES.each do |reg_ex|
42
+ msg = msg.gsub(reg_ex){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
42
43
  end
43
44
  end
44
- original_formatter.call(severity, datetime, progname, msg)
45
+ original_formatter.call(severity, date_time, program_name, msg)
45
46
  end
46
47
  end
47
48
 
48
49
  def secret?(keyword, value)
49
50
  keyword = keyword.to_s if keyword.is_a?(Symbol)
50
51
  # only Strings can be secrets, not booleans, or hash, arrays
51
- keyword.is_a?(String) && ALL_SECRETS.any?{|kw|keyword.include?(kw)} && value.is_a?(String)
52
+ return false unless keyword.is_a?(String) && value.is_a?(String)
53
+ # those are not secrets
54
+ return false if FALSE_POSITIVES.any?{|f|f.match?(keyword)}
55
+ # check if keyword (name) contains an element that designate it as a secret
56
+ ALL_SECRETS.any?{|kw|keyword.include?(kw)}
52
57
  end
53
58
 
54
59
  def deep_remove_secret(obj, is_name_value: false)
data/lib/aspera/ssh.rb CHANGED
@@ -44,7 +44,7 @@ module Aspera
44
44
  end
45
45
  raise error_message
46
46
  end
47
- # send command to SSH channel (execute)
47
+ # send command to SSH channel (execute) cspell: disable-next-line
48
48
  channel.send('cexe'.reverse, cmd){|_ch, _success|channel.send_data(input) unless input.nil?}
49
49
  end
50
50
  # wait for channel to finish (command exit)
data/lib/aspera/sync.rb CHANGED
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # cspell:words logdir
4
+
3
5
  require 'aspera/command_line_builder'
4
6
  require 'aspera/fasp/installation'
5
7
  require 'json'
@@ -8,6 +10,13 @@ require 'base64'
8
10
  module Aspera
9
11
  # builds command line arg for async
10
12
  class Sync
13
+ # default is push
14
+ DIRECTIONS = %i[push pull bidi].freeze
15
+ DIRECTION_TO_REQUEST_TYPE = {
16
+ push: :sync_upload,
17
+ pull: :sync_download,
18
+ bidi: :sync
19
+ }.freeze
11
20
  PARAMS_VX_INSTANCE =
12
21
  {
13
22
  'alt_logdir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
@@ -38,7 +47,7 @@ module Aspera
38
47
  'cooloff' => { cli: { type: :opt_with_arg}, accepted_types: :int},
39
48
  'pending_max' => { cli: { type: :opt_with_arg}, accepted_types: :int},
40
49
  'scan_intensity' => { cli: { type: :opt_with_arg}, accepted_types: :string},
41
- 'cipher' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: true},
50
+ 'cipher' => { cli: { type: :opt_with_arg, convert: 'Aspera::Fasp::Parameters.convert_remove_hyphen'}, accepted_types: :string, ts: true},
42
51
  'transfer_threads' => { cli: { type: :opt_with_arg}, accepted_types: :int},
43
52
  'preserve_time' => { cli: { type: :opt_without_arg}, ts: :preserve_times},
44
53
  'preserve_access_time' => { cli: { type: :opt_without_arg}, ts: nil},
@@ -60,7 +69,7 @@ module Aspera
60
69
  PARAMS_VX_KEYS = %w[instance sessions].freeze
61
70
 
62
71
  # new API
63
- TS_TO_PARAMS = {
72
+ TS_TO_PARAMS_V2 = {
64
73
  'remote_host' => 'remote.host',
65
74
  'remote_user' => 'remote.user',
66
75
  'remote_password' => 'remote.pass',
@@ -74,14 +83,27 @@ module Aspera
74
83
 
75
84
  ASYNC_EXECUTABLE = 'async'
76
85
 
77
- private_constant :PARAMS_VX_INSTANCE, :PARAMS_VX_SESSION, :PARAMS_VX_KEYS, :ASYNC_EXECUTABLE
86
+ private_constant :PARAMS_VX_INSTANCE, :PARAMS_VX_SESSION, :PARAMS_VX_KEYS, :ASYNC_EXECUTABLE, :TS_TO_PARAMS_V2
87
+
88
+ attr_reader :env_args
78
89
 
79
- class << self
80
- def update_parameters_with_transfer_spec(sync_params, transfer_spec)
81
- if sync_params.key?('local')
90
+ # @param sync_params [Hash] sync parameters, old or new format
91
+ # @param node_sync [Object|nil]
92
+ def initialize(sync_params, node_sync)
93
+ raise StandardError, 'parameter must be Hash' unless sync_params.is_a?(Hash)
94
+ raise 'node_sync misses method transfer_spec' unless node_sync.nil? || node_sync.respond_to?(:transfer_spec)
95
+ @env_args = {
96
+ args: [],
97
+ env: {}
98
+ }
99
+ if sync_params.key?('local')
100
+ # async native JSON format (v2)
101
+ raise StandardError, 'remote must be Hash' unless sync_params['remote'].is_a?(Hash)
102
+ unless node_sync.nil?
103
+ transfer_spec = node_sync.transfer_spec(sync_params['direction'], sync_params['local']['path'], sync_params['remote']['path'])
82
104
  # async native JSON format
83
105
  raise StandardError, 'local must be Hash' unless sync_params['local'].is_a?(Hash)
84
- TS_TO_PARAMS.each do |ts_param, sy_path|
106
+ TS_TO_PARAMS_V2.each do |ts_param, sy_path|
85
107
  next unless transfer_spec.key?(ts_param)
86
108
  sy_dig = sy_path.split('.')
87
109
  param = sy_dig.pop
@@ -93,38 +115,23 @@ module Aspera
93
115
  sync_params['remote']['connect_mode'] ||= sync_params['remote'].key?('ws_port') ? 'ws' : 'ssh'
94
116
  sync_params['remote']['private_key_paths'] ||= Fasp::Installation.instance.bypass_keys if transfer_spec.key?('token')
95
117
  sync_params['remote']['path'] ||= '/' if transfer_spec.dig(*%w[tags aspera node file_id])
96
- elsif sync_params.key?('sessions')
118
+ end
119
+ @env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"]
120
+ elsif sync_params.key?('sessions')
121
+ # ascli JSON format (v1)
122
+ unless node_sync.nil?
97
123
  sync_params['sessions'].each do |session|
98
- PARAMS_VX_SESSION.each do |async_param, behaviour|
99
- if behaviour.key?(:ts)
100
- tspec_param = behaviour[:ts].is_a?(TrueClass) ? async_param : behaviour[:ts].to_s
124
+ transfer_spec = node_sync.transfer_spec(session['direction'], session['local_dir'], session['remote_dir'])
125
+ PARAMS_VX_SESSION.each do |async_param, behavior|
126
+ if behavior.key?(:ts)
127
+ tspec_param = behavior[:ts].is_a?(TrueClass) ? async_param : behavior[:ts].to_s
101
128
  session[async_param] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
102
129
  end
103
130
  end
104
131
  session['private_key_paths'] = Fasp::Installation.instance.bypass_keys if transfer_spec.key?('token')
105
132
  session['remote_dir'] = '/' if transfer_spec.dig(*%w[tags aspera node file_id])
106
133
  end
107
- else
108
- raise 'At least one of `local` or `sessions` must be present in async parameters'
109
134
  end
110
- Log.dump(:sync, sync_params)
111
- end
112
- end
113
-
114
- attr_reader :env_args
115
-
116
- def initialize(sync_params)
117
- raise StandardError, 'parameter must be Hash' unless sync_params.is_a?(Hash)
118
- @env_args = {
119
- args: [],
120
- env: {}
121
- }
122
- if sync_params.key?('local')
123
- # async native JSON format
124
- raise StandardError, 'remote must be Hash' unless sync_params['remote'].is_a?(Hash)
125
- @env_args[:args] = "--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"
126
- elsif sync_params.key?('sessions')
127
- # ascli JSON format
128
135
  raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
129
136
  sync_params.keys.push('instance').uniq.sort.eql?(PARAMS_VX_KEYS)
130
137
  raise StandardError, 'sessions key must be Array' unless sync_params['sessions'].is_a?(Array)
@@ -139,7 +146,7 @@ module Aspera
139
146
 
140
147
  sync_params['sessions'].each do |session_params|
141
148
  raise StandardError, 'sessions must contain hashes' unless session_params.is_a?(Hash)
142
- raise StandardError, 'session must contain at leat name' unless session_params.key?('name')
149
+ raise StandardError, 'session must contain at least name' unless session_params.key?('name')
143
150
  session_builder = Aspera::CommandLineBuilder.new(session_params, PARAMS_VX_SESSION)
144
151
  session_builder.process_params
145
152
  session_builder.add_env_args(@env_args[:env], @env_args[:args])
@@ -147,6 +154,7 @@ module Aspera
147
154
  else
148
155
  raise 'At least one of `local` or `sessions` must be present in async parameters'
149
156
  end
157
+ Log.dump(:sync, sync_params)
150
158
  end
151
159
 
152
160
  def start
@@ -160,7 +168,7 @@ module Aspera
160
168
  else raise 'internal error: unspecified case'
161
169
  end
162
170
  end
163
- end
171
+ end # end Sync
164
172
 
165
173
  class SyncAdmin
166
174
  ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
@@ -8,24 +8,26 @@ require 'openssl'
8
8
  module Aspera
9
9
  class WebServerSimple < WEBrick::HTTPServer
10
10
  CERT_PARAMETERS = %i[key cert chain].freeze
11
- # generates and adds self signed cert to provided webrick options
12
- def self.fill_self_signed_cert(cert, key)
13
- cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/C=FR/O=Test/OU=Test/CN=Test')
14
- cert.not_before = Time.now
15
- cert.not_after = Time.now + 365 * 24 * 60 * 60
16
- cert.public_key = key.public_key
17
- cert.serial = 0x0
18
- cert.version = 2
19
- ef = OpenSSL::X509::ExtensionFactory.new
20
- ef.issuer_certificate = cert
21
- ef.subject_certificate = cert
22
- cert.extensions = [
23
- ef.create_extension('basicConstraints', 'CA:TRUE', true),
24
- ef.create_extension('subjectKeyIdentifier', 'hash')
25
- # ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
26
- ]
27
- cert.add_extension(ef.create_extension('authorityKeyIdentifier', 'keyid:always,issuer:always'))
28
- cert.sign(key, OpenSSL::Digest.new('SHA256'))
11
+ class << self
12
+ # generates and adds self signed cert to provided webrick options
13
+ def fill_self_signed_cert(cert, key)
14
+ cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/C=FR/O=Test/OU=Test/CN=Test')
15
+ cert.not_before = Time.now
16
+ cert.not_after = Time.now + 365 * 24 * 60 * 60
17
+ cert.public_key = key.public_key
18
+ cert.serial = 0x0
19
+ cert.version = 2
20
+ ef = OpenSSL::X509::ExtensionFactory.new
21
+ ef.issuer_certificate = cert
22
+ ef.subject_certificate = cert
23
+ cert.extensions = [
24
+ ef.create_extension('basicConstraints', 'CA:TRUE', true),
25
+ ef.create_extension('subjectKeyIdentifier', 'hash')
26
+ # ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
27
+ ]
28
+ cert.add_extension(ef.create_extension('authorityKeyIdentifier', 'keyid:always,issuer:always'))
29
+ cert.sign(key, OpenSSL::Digest.new('SHA256'))
30
+ end
29
31
  end
30
32
 
31
33
  # @param uri [URI]
@@ -66,6 +68,8 @@ module Aspera
66
68
  end
67
69
  # self signed certificate generates characters on STDERR, see create_self_signed_cert in webrick/ssl.rb
68
70
  Log.capture_stderr { super(webrick_options) }
71
+ # kill -USR1 for graceful shutdown
72
+ Kernel.trap('USR1') { shutdown }
69
73
  end
70
74
 
71
75
  # log web server access ( option AccessLog )
data.tar.gz.sig CHANGED
Binary file