aspera-cli 4.12.0 → 4.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +45 -5
- data/CONTRIBUTING.md +113 -22
- data/README.md +1289 -754
- data/bin/ascli +3 -3
- data/examples/dascli +1 -1
- data/examples/rubyc +24 -0
- data/lib/aspera/aoc.rb +63 -74
- data/lib/aspera/ascmd.rb +5 -3
- data/lib/aspera/cli/basic_auth_plugin.rb +6 -6
- data/lib/aspera/cli/extended_value.rb +24 -37
- data/lib/aspera/cli/formatter.rb +23 -25
- data/lib/aspera/cli/info.rb +2 -4
- data/lib/aspera/cli/main.rb +27 -27
- data/lib/aspera/cli/manager.rb +143 -120
- data/lib/aspera/cli/plugin.rb +88 -43
- data/lib/aspera/cli/plugins/alee.rb +2 -2
- data/lib/aspera/cli/plugins/aoc.rb +235 -104
- data/lib/aspera/cli/plugins/ats.rb +16 -18
- data/lib/aspera/cli/plugins/bss.rb +3 -3
- data/lib/aspera/cli/plugins/config.rb +190 -373
- data/lib/aspera/cli/plugins/console.rb +4 -6
- data/lib/aspera/cli/plugins/cos.rb +12 -13
- data/lib/aspera/cli/plugins/faspex.rb +21 -21
- data/lib/aspera/cli/plugins/faspex5.rb +399 -150
- data/lib/aspera/cli/plugins/node.rb +260 -174
- data/lib/aspera/cli/plugins/orchestrator.rb +15 -18
- data/lib/aspera/cli/plugins/preview.rb +40 -62
- data/lib/aspera/cli/plugins/server.rb +33 -16
- data/lib/aspera/cli/plugins/shares.rb +24 -33
- data/lib/aspera/cli/plugins/sync.rb +6 -6
- data/lib/aspera/cli/transfer_agent.rb +47 -30
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +9 -7
- data/lib/aspera/command_line_builder.rb +2 -1
- data/lib/aspera/cos_node.rb +1 -1
- data/lib/aspera/data/6 +0 -0
- data/lib/aspera/environment.rb +7 -3
- data/lib/aspera/fasp/agent_connect.rb +6 -1
- data/lib/aspera/fasp/agent_direct.rb +17 -17
- data/lib/aspera/fasp/agent_httpgw.rb +138 -60
- data/lib/aspera/fasp/agent_node.rb +14 -4
- data/lib/aspera/fasp/agent_trsdk.rb +2 -0
- data/lib/aspera/fasp/error_info.rb +2 -0
- data/lib/aspera/fasp/installation.rb +19 -19
- data/lib/aspera/fasp/parameters.rb +29 -20
- data/lib/aspera/fasp/parameters.yaml +5 -2
- data/lib/aspera/fasp/resume_policy.rb +3 -3
- data/lib/aspera/fasp/transfer_spec.rb +8 -5
- data/lib/aspera/fasp/uri.rb +23 -21
- data/lib/aspera/faspex_gw.rb +1 -0
- data/lib/aspera/faspex_postproc.rb +3 -3
- data/lib/aspera/hash_ext.rb +12 -2
- data/lib/aspera/keychain/macos_security.rb +13 -13
- data/lib/aspera/log.rb +1 -0
- data/lib/aspera/node.rb +73 -84
- data/lib/aspera/oauth.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +1 -1
- data/lib/aspera/preview/file_types.rb +8 -6
- data/lib/aspera/preview/generator.rb +23 -11
- data/lib/aspera/preview/options.rb +3 -2
- data/lib/aspera/preview/terminal.rb +80 -0
- data/lib/aspera/preview/utils.rb +11 -11
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/rest.rb +42 -4
- data/lib/aspera/rest_call_error.rb +3 -1
- data/lib/aspera/secret_hider.rb +10 -5
- data/lib/aspera/ssh.rb +1 -1
- data/lib/aspera/sync.rb +41 -33
- data/lib/aspera/web_server_simple.rb +22 -18
- data.tar.gz.sig +0 -0
- metadata +40 -48
- metadata.gz.sig +0 -0
- data/docs/test_env.conf +0 -179
- data/examples/aoc.rb +0 -30
- data/examples/faspex4.rb +0 -94
- data/examples/node.rb +0 -96
- data/examples/server.rb +0 -93
- data/lib/aspera/data/7 +0 -0
@@ -1,5 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# ffmpeg options:
|
4
|
+
# spellchecker:ignore pauseframes libx264 trunc bufsize muxer apng libmp3lame maxrate posterize movflags faststart
|
5
|
+
# spellchecker:ignore palettegen paletteuse pointsize bordercolor repage lanczos
|
6
|
+
|
3
7
|
require 'open3'
|
4
8
|
require 'aspera/preview/options'
|
5
9
|
require 'aspera/preview/utils'
|
@@ -12,6 +16,8 @@ module Aspera
|
|
12
16
|
# values for preview_format : output format
|
13
17
|
PREVIEW_FORMATS = %i[png mp4].freeze
|
14
18
|
|
19
|
+
FFMPEG_OPTIONS_LIST = %w[in out].freeze
|
20
|
+
|
15
21
|
# CLI needs to know conversion type to know if need skip it
|
16
22
|
attr_reader :conversion_type
|
17
23
|
|
@@ -22,7 +28,6 @@ module Aspera
|
|
22
28
|
# supported preview type is one of Preview::PREVIEW_FORMATS
|
23
29
|
# the resulting preview file type is taken from destination file extension.
|
24
30
|
# conversion methods are provided by private methods: convert_<conversion_type>_to_<preview_format>
|
25
|
-
# (combi = combination of source file type and destination format)
|
26
31
|
# -> conversion_type is one of FileTypes::CONVERSION_TYPES
|
27
32
|
# -> preview_format is one of Generator::PREVIEW_FORMATS
|
28
33
|
# the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
|
@@ -70,10 +75,10 @@ module Aspera
|
|
70
75
|
# check that generated size does not exceed maximum
|
71
76
|
result_size = File.size(@destination_file_path)
|
72
77
|
if result_size > @options.max_size
|
73
|
-
Log.log.warn{"preview size exceeds maximum #{result_size} > #{@options.max_size}"}
|
78
|
+
Log.log.warn{"preview size exceeds maximum allowed #{result_size} > #{@options.max_size}"}
|
74
79
|
end
|
75
80
|
rescue StandardError => e
|
76
|
-
Log.log.error{"
|
81
|
+
Log.log.error{"Ignoring: #{e.message}"}
|
77
82
|
Log.log.debug(e.backtrace.join("\n").red)
|
78
83
|
FileUtils.cp(File.expand_path(@preview_format_symb.eql?(:mp4) ? 'video_error.png' : 'image_error.png', File.dirname(__FILE__)), @destination_file_path)
|
79
84
|
ensure
|
@@ -102,11 +107,11 @@ module Aspera
|
|
102
107
|
def convert_video_to_mp4_using_blend
|
103
108
|
p_duration = Utils.video_get_duration(@source_file_path)
|
104
109
|
p_start_offset = @options.video_start_sec.to_i
|
105
|
-
|
110
|
+
p_key_frame_count = @options.blend_keyframes.to_i
|
106
111
|
last_keyframe = nil
|
107
112
|
current_index = 1
|
108
|
-
1.upto(
|
109
|
-
offset_seconds = get_offset(p_duration, p_start_offset,
|
113
|
+
1.upto(p_key_frame_count) do |i|
|
114
|
+
offset_seconds = get_offset(p_duration, p_start_offset, p_key_frame_count, i)
|
110
115
|
Utils.video_dump_frame(@source_file_path, offset_seconds, @options.video_scale, this_tmpdir, current_index)
|
111
116
|
Utils.video_dupe_frame(this_tmpdir, current_index, @options.blend_pauseframes)
|
112
117
|
Utils.video_blend_frames(this_tmpdir, last_keyframe, current_index) unless last_keyframe.nil?
|
@@ -133,17 +138,17 @@ module Aspera
|
|
133
138
|
File.open(filelist, 'w+') do |f|
|
134
139
|
1.upto(@options.clips_count.to_i) do |i|
|
135
140
|
offset_seconds = get_offset(p_duration, @options.video_start_sec.to_i, @options.clips_count.to_i, i)
|
136
|
-
|
141
|
+
tmp_file_name = format('clip%04d.mp4', i)
|
137
142
|
Utils.ffmpeg(
|
138
143
|
in_f: @source_file_path,
|
139
144
|
in_p: ['-ss', offset_seconds * 0.9],
|
140
|
-
out_f: File.join(this_tmpdir,
|
145
|
+
out_f: File.join(this_tmpdir, tmp_file_name),
|
141
146
|
out_p: [
|
142
147
|
'-ss', offset_seconds * 0.1,
|
143
148
|
'-t', @options.clips_length,
|
144
149
|
'-filter:v', "scale=#{@options.video_scale}",
|
145
150
|
'-codec:a', 'libmp3lame'])
|
146
|
-
f.puts("file '#{
|
151
|
+
f.puts("file '#{tmp_file_name}'")
|
147
152
|
end
|
148
153
|
end
|
149
154
|
# concat clips
|
@@ -154,12 +159,19 @@ module Aspera
|
|
154
159
|
out_p: ['-codec', 'copy'])
|
155
160
|
end
|
156
161
|
|
157
|
-
# do a simple
|
162
|
+
# do a simple re-encoding
|
158
163
|
def convert_video_to_mp4_using_reencode
|
164
|
+
options = @options.reencode_ffmpeg
|
165
|
+
raise 'reencode_ffmpeg must be a Hash' unless options.is_a?(Hash)
|
166
|
+
options.each do |k, v|
|
167
|
+
raise "Key not supported: #{k}. Use keys: #{FFMPEG_OPTIONS_LIST.join(',')}" unless FFMPEG_OPTIONS_LIST.include?(k)
|
168
|
+
raise "Value for #{k} must be Array" unless v.is_a?(Array)
|
169
|
+
end
|
159
170
|
Utils.ffmpeg(
|
160
171
|
in_f: @source_file_path,
|
172
|
+
in_p: options['in'] || ['-ss', @options.video_start_sec.to_i * 0.9],
|
161
173
|
out_f: @destination_file_path,
|
162
|
-
out_p: [
|
174
|
+
out_p: options['out'] || [
|
163
175
|
'-t', '60',
|
164
176
|
'-codec:v', 'libx264',
|
165
177
|
'-profile:v', 'high',
|
@@ -20,8 +20,9 @@ module Aspera
|
|
20
20
|
{ name: :thumb_text_font, default: 'Courier', description: 'png: plaintext: font to render text with imagemagick convert (identify -list font)'},
|
21
21
|
{ name: :video_conversion, default: :reencode, description: 'mp4: method for preview generation', values: VIDEO_CONVERSION_METHODS },
|
22
22
|
{ name: :video_png_conv, default: :fixed, description: 'mp4: method for thumbnail generation', values: VIDEO_THUMBNAIL_METHODS },
|
23
|
-
{ name: :
|
24
|
-
{ name: :
|
23
|
+
{ name: :video_scale, default: "'min(iw,360)':-2", description: 'mp4: all: video scale (ffmpeg)' },
|
24
|
+
{ name: :video_start_sec, default: 10, description: 'mp4: all: start offset (seconds) of video preview' },
|
25
|
+
{ name: :reencode_ffmpeg, default: {}, description: 'mp4: reencode: options to ffmpeg' },
|
25
26
|
{ name: :blend_keyframes, default: 30, description: 'mp4: blend: # key frames' },
|
26
27
|
{ name: :blend_pauseframes, default: 3, description: 'mp4: blend: # pause frames' },
|
27
28
|
{ name: :blend_transframes, default: 5, description: 'mp4: blend: # transition blend frames' },
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# cspell:words Magick MAGICKCORE ITERM mintty winsize termcap
|
4
|
+
|
5
|
+
require 'rmagick' # https://rmagick.github.io/index.html
|
6
|
+
require 'rainbow'
|
7
|
+
require 'io/console'
|
8
|
+
module Aspera
|
9
|
+
module Preview
|
10
|
+
# Generates a string that can display an image in a terminal
|
11
|
+
class Terminal
|
12
|
+
# quantum depth is 8 or 16: convert xc: -format "%q" info:
|
13
|
+
# Rainbow only supports 8-bit colors
|
14
|
+
SHIFT_FOR_8_BIT = Magick::MAGICKCORE_QUANTUM_DEPTH - 8
|
15
|
+
ITERM_NAMES = %w[iTerm WezTerm mintty].freeze
|
16
|
+
TERM_ENV_VARS = %w[TERM_PROGRAM LC_TERMINAL].freeze
|
17
|
+
private_constant :SHIFT_FOR_8_BIT, :ITERM_NAMES, :TERM_ENV_VARS
|
18
|
+
class << self
|
19
|
+
def build(blob, reserved_lines: 0, double_precision: true)
|
20
|
+
return iterm_display_image(blob) if iterm_supported?
|
21
|
+
image = Magick::ImageList.new.from_blob(blob)
|
22
|
+
(term_rows, term_columns) = IO.console.winsize
|
23
|
+
term_rows -= reserved_lines
|
24
|
+
# compute scaling to fit terminal
|
25
|
+
fit_term_ratio = [term_rows / image.rows.to_f, term_columns / image.columns.to_f].min
|
26
|
+
# TODO: retrieve terminal font ratio using some termcap ?
|
27
|
+
font_ratio = 1.7
|
28
|
+
height_ratio = double_precision ? 2.0 : 1.0
|
29
|
+
image = image.scale((image.columns * fit_term_ratio * font_ratio).to_i, (image.rows * fit_term_ratio * height_ratio).to_i)
|
30
|
+
# get all pixel colors, adjusted for Rainbow
|
31
|
+
pixel_colors = []
|
32
|
+
image.each_pixel do |pixel, col, row|
|
33
|
+
pixel_rgb = [pixel.red, pixel.green, pixel.blue]
|
34
|
+
pixel_rgb = pixel_rgb.map { |color| color >> SHIFT_FOR_8_BIT } unless SHIFT_FOR_8_BIT.eql?(0)
|
35
|
+
# init 2-dim array
|
36
|
+
pixel_colors[row] ||= []
|
37
|
+
pixel_colors[row][col] = pixel_rgb
|
38
|
+
end
|
39
|
+
# now generate text
|
40
|
+
text_pixels = []
|
41
|
+
pixel_colors.each_with_index do |row_data, row|
|
42
|
+
next if double_precision && row.odd?
|
43
|
+
row_data.each_with_index do |pixel_rgb, col|
|
44
|
+
text_pixels.push("\n") if col.eql?(0) && !row.eql?(0)
|
45
|
+
if double_precision
|
46
|
+
text_pixels.push(Rainbow('▄').background(pixel_rgb).foreground(pixel_colors[row + 1][col]))
|
47
|
+
else
|
48
|
+
text_pixels.push(Rainbow(' ').background(pixel_rgb))
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
return text_pixels.join
|
53
|
+
end
|
54
|
+
|
55
|
+
# display image in iTerm2
|
56
|
+
def iterm_display_image(blob)
|
57
|
+
# image = Magick::ImageList.new.from_blob(blob)
|
58
|
+
arguments = {
|
59
|
+
inline: 1,
|
60
|
+
preserveAspectRatio: 1,
|
61
|
+
size: blob.length
|
62
|
+
# width: image.columns,
|
63
|
+
# height: image.rows
|
64
|
+
}.map { |k, v| "#{k}=#{v}" }.join(';')
|
65
|
+
# \a is BEL, \e is ESC : https://github.com/ruby/ruby/blob/master/doc/syntax/literals.rdoc#label-Strings
|
66
|
+
# https://iterm2.com/documentation-images.html
|
67
|
+
return "\e]1337;File=#{arguments}:#{Base64.encode64(blob)}\a"
|
68
|
+
end
|
69
|
+
|
70
|
+
# @return [Boolean] true if the terminal supports iTerm2 image display
|
71
|
+
def iterm_supported?
|
72
|
+
TERM_ENV_VARS.each do |env_var|
|
73
|
+
return true if ITERM_NAMES.any? { |term| ENV[env_var]&.include?(term) }
|
74
|
+
end
|
75
|
+
false
|
76
|
+
end
|
77
|
+
end # class << self
|
78
|
+
end # class Terminal
|
79
|
+
end # module Preview
|
80
|
+
end # module Aspera
|
data/lib/aspera/preview/utils.rb
CHANGED
@@ -14,12 +14,12 @@ module Aspera
|
|
14
14
|
# shell exit code when command is not found
|
15
15
|
BASH_EXIT_NOT_FOUND = 127
|
16
16
|
# external binaries used
|
17
|
-
|
18
|
-
|
19
|
-
private_constant :BASH_SPECIAL_CHARACTERS, :BASH_EXIT_NOT_FOUND, :
|
17
|
+
EXTERNAL_TOOLS = %i[ffmpeg ffprobe convert composite optipng unoconv].freeze
|
18
|
+
TEMP_FORMAT = 'img%04d.jpg'
|
19
|
+
private_constant :BASH_SPECIAL_CHARACTERS, :BASH_EXIT_NOT_FOUND, :EXTERNAL_TOOLS, :TEMP_FORMAT
|
20
20
|
|
21
21
|
class << self
|
22
|
-
# returns string with single quotes suitable for bash if there is any bash
|
22
|
+
# returns string with single quotes suitable for bash if there is any bash meta-character
|
23
23
|
def shell_quote(argument)
|
24
24
|
return argument unless argument.chars.any?{|c|BASH_SPECIAL_CHARACTERS.include?(c)}
|
25
25
|
return "'" + argument.gsub(/'/){|_s| "'\"'\"'"} + "'"
|
@@ -27,7 +27,7 @@ module Aspera
|
|
27
27
|
|
28
28
|
# check that external tools can be executed
|
29
29
|
def check_tools(skip_types=[])
|
30
|
-
tools_to_check =
|
30
|
+
tools_to_check = EXTERNAL_TOOLS.dup
|
31
31
|
tools_to_check.delete(:unoconv) if skip_types.include?(:office)
|
32
32
|
# Check for binaries
|
33
33
|
tools_to_check.each do |command_symb|
|
@@ -39,7 +39,7 @@ module Aspera
|
|
39
39
|
# one could use "system", but we would need to redirect stdout/err
|
40
40
|
# @return true if su
|
41
41
|
def external_command(command_symb, command_args)
|
42
|
-
raise "unexpected command #{command_symb}" unless
|
42
|
+
raise "unexpected command #{command_symb}" unless EXTERNAL_TOOLS.include?(command_symb)
|
43
43
|
# build command line, and quote special characters
|
44
44
|
command = command_args.clone.unshift(command_symb).map{|i| shell_quote(i.to_s)}.join(' ')
|
45
45
|
Log.log.debug{"cmd=#{command}".blue}
|
@@ -55,7 +55,7 @@ module Aspera
|
|
55
55
|
raise "Error: #{command_symb} is not in the PATH"
|
56
56
|
end
|
57
57
|
unless exit_status.success?
|
58
|
-
Log.log.error{"
|
58
|
+
Log.log.error{"command line: #{command}"}
|
59
59
|
Log.log.error{"Error code: #{exit_status}"}
|
60
60
|
Log.log.error{"stdout: #{stdout}"}
|
61
61
|
Log.log.error{"stderr: #{stderr}"}
|
@@ -69,7 +69,7 @@ module Aspera
|
|
69
69
|
# input_file,input_args,output_file,output_args
|
70
70
|
a[:gl_p] ||= [
|
71
71
|
'-y', # overwrite output without asking
|
72
|
-
'-loglevel', 'error' # show only errors and up
|
72
|
+
'-loglevel', 'error' # show only errors and up
|
73
73
|
]
|
74
74
|
a[:in_p] ||= []
|
75
75
|
a[:out_p] ||= []
|
@@ -82,17 +82,17 @@ module Aspera
|
|
82
82
|
result = external_command(:ffprobe, [
|
83
83
|
'-loglevel', 'error',
|
84
84
|
'-show_entries', 'format=duration',
|
85
|
-
'-print_format', 'default=noprint_wrappers=1:nokey=1',
|
85
|
+
'-print_format', 'default=noprint_wrappers=1:nokey=1', # cspell:disable-line
|
86
86
|
input_file])
|
87
87
|
return result[:stdout].to_f
|
88
88
|
end
|
89
89
|
|
90
90
|
def ffmpeg_fmt(temp_folder)
|
91
|
-
return File.join(temp_folder,
|
91
|
+
return File.join(temp_folder, TEMP_FORMAT)
|
92
92
|
end
|
93
93
|
|
94
94
|
def get_tmp_num_filepath(temp_folder, file_number)
|
95
|
-
return File.join(temp_folder, format(
|
95
|
+
return File.join(temp_folder, format(TEMP_FORMAT, file_number))
|
96
96
|
end
|
97
97
|
|
98
98
|
def video_dupe_frame(temp_folder, index, count)
|
@@ -70,7 +70,7 @@ function shExpMatch(str, shell_expr) {
|
|
70
70
|
}
|
71
71
|
function weekdayRange(wd1, wd2, gmt) {
|
72
72
|
var today = new Date();
|
73
|
-
var days = 'SUNMONTUEWEDTHUFRISAT';
|
73
|
+
var days = 'SUNMONTUEWEDTHUFRISAT'; // cspell: disable-line
|
74
74
|
wd1 = wd1.toUpperCase();
|
75
75
|
if (wd2 == undefined)
|
76
76
|
wd2 = wd1;
|
@@ -138,7 +138,7 @@ function dateRange() {
|
|
138
138
|
else
|
139
139
|
return false;
|
140
140
|
} else if (typeof arg == 'string') {
|
141
|
-
var months = 'JANFEBMARAPRMAYJUNJULAUGSEPOCTNOVDEC';
|
141
|
+
var months = 'JANFEBMARAPRMAYJUNJULAUGSEPOCTNOVDEC'; // cspell: disable-line
|
142
142
|
arg = arg.toUpperCase();
|
143
143
|
arg = months.indexOf(arg);
|
144
144
|
if (arg == -1)
|
data/lib/aspera/rest.rb
CHANGED
@@ -38,6 +38,11 @@ module Aspera
|
|
38
38
|
|
39
39
|
ARRAY_PARAMS = '[]'
|
40
40
|
|
41
|
+
private_constant :ARRAY_PARAMS
|
42
|
+
|
43
|
+
# error message when entity not found
|
44
|
+
ENTITY_NOT_FOUND = 'No such'
|
45
|
+
|
41
46
|
class << self
|
42
47
|
# define accessors
|
43
48
|
@@global.each_key do |p|
|
@@ -50,6 +55,12 @@ module Aspera
|
|
50
55
|
|
51
56
|
def basic_creds(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
|
52
57
|
|
58
|
+
# used to build a parameter list prefixed with "[]"
|
59
|
+
# @param values [Array] list of values
|
60
|
+
def array_params(values)
|
61
|
+
return [ARRAY_PARAMS].concat(values)
|
62
|
+
end
|
63
|
+
|
53
64
|
# build URI from URL and parameters and check it is http or https
|
54
65
|
def build_uri(url, params=nil)
|
55
66
|
uri = URI.parse(url)
|
@@ -224,8 +235,9 @@ module Aspera
|
|
224
235
|
begin
|
225
236
|
# we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
|
226
237
|
oauth_tries ||= 2
|
227
|
-
|
228
|
-
|
238
|
+
# initialize with number of initial retries allowed, nil gives zero
|
239
|
+
tries_remain_redirect = call_data[:redirect_max].to_i if tries_remain_redirect.nil?
|
240
|
+
Log.log.debug("send request (retries=#{tries_remain_redirect})")
|
229
241
|
# make http request (pipelined)
|
230
242
|
http_session.request(req) do |response|
|
231
243
|
result[:http] = response
|
@@ -286,8 +298,8 @@ module Aspera
|
|
286
298
|
Log.log.debug{"using new token=#{call_data[:headers]['Authorization']}"}
|
287
299
|
retry unless (oauth_tries -= 1).zero?
|
288
300
|
end # if oauth
|
289
|
-
#
|
290
|
-
if e.response.is_a?(Net::HTTPRedirection)
|
301
|
+
# redirect ? (any code beginning with 3)
|
302
|
+
if tries_remain_redirect.positive? && e.response.is_a?(Net::HTTPRedirection)
|
291
303
|
tries_remain_redirect -= 1
|
292
304
|
current_uri = URI.parse(call_data[:base_url])
|
293
305
|
new_url = e.response['location']
|
@@ -336,5 +348,31 @@ module Aspera
|
|
336
348
|
def cancel(subpath)
|
337
349
|
return call({operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'}})
|
338
350
|
end
|
351
|
+
|
352
|
+
# Query by name and returns a single result, else it throws an exception (no or multiple results)
|
353
|
+
# @param subpath path of entity in API
|
354
|
+
# @param search_name name of searched entity
|
355
|
+
# @param options additional search options
|
356
|
+
def lookup_by_name(subpath, search_name, options={})
|
357
|
+
# returns entities whose name contains value (case insensitive)
|
358
|
+
matching_items = read(subpath, options.merge({'q' => CGI.escape(search_name)}))[:data]
|
359
|
+
# API style: {totalcount:, ...} cspell: disable-line
|
360
|
+
# TODO: not generic enough ? move somewhere ? inheritance ?
|
361
|
+
matching_items = matching_items[subpath] if matching_items.is_a?(Hash)
|
362
|
+
raise "Internal error: expecting array, have #{matching_items.class}" unless matching_items.is_a?(Array)
|
363
|
+
case matching_items.length
|
364
|
+
when 1 then return matching_items.first
|
365
|
+
when 0 then raise %Q{#{ENTITY_NOT_FOUND} #{subpath}: "#{search_name}"}
|
366
|
+
else
|
367
|
+
# multiple case insensitive partial matches, try case insensitive full match
|
368
|
+
# (anyway AoC does not allow creation of 2 entities with same case insensitive name)
|
369
|
+
name_matches = matching_items.select{|i|i['name'].casecmp?(search_name)}
|
370
|
+
case name_matches.length
|
371
|
+
when 1 then return name_matches.first
|
372
|
+
when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{|i|i['name']}} but no case insensitive full match. Please be more specific or give exact name.) # rubocop:disable Layout/LineLength
|
373
|
+
else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{|i|i['name']}}"
|
374
|
+
end
|
375
|
+
end
|
376
|
+
end
|
339
377
|
end
|
340
378
|
end # module Aspera
|
@@ -5,7 +5,9 @@ module Aspera
|
|
5
5
|
class RestCallError < StandardError
|
6
6
|
attr_accessor :request, :response
|
7
7
|
|
8
|
-
# @param
|
8
|
+
# @param req HTTP Request object
|
9
|
+
# @param resp HTTP Response object
|
10
|
+
# @param msg Error message
|
9
11
|
def initialize(req, resp, msg)
|
10
12
|
@request = req
|
11
13
|
@response = resp
|
data/lib/aspera/secret_hider.rb
CHANGED
@@ -12,6 +12,7 @@ module Aspera
|
|
12
12
|
# keys in hash that contain secrets
|
13
13
|
KEY_SECRETS = %w[password secret passphrase _key apikey crn token].freeze
|
14
14
|
ALL_SECRETS = [ASCP_ENV_SECRETS, KEY_SECRETS].flatten.freeze
|
15
|
+
FALSE_POSITIVES = [/^access_key$/].freeze
|
15
16
|
# regex that define named captures :begin and :end
|
16
17
|
REGEX_LOG_REPLACES = [
|
17
18
|
# CLI manager get/set options
|
@@ -35,20 +36,24 @@ module Aspera
|
|
35
36
|
def log_formatter(original_formatter)
|
36
37
|
original_formatter ||= Logger::Formatter.new
|
37
38
|
# NOTE: that @log_secrets may be set AFTER this init is done, so it's done at runtime
|
38
|
-
return lambda do |severity,
|
39
|
+
return lambda do |severity, date_time, program_name, msg|
|
39
40
|
if msg.is_a?(String) && !@log_secrets
|
40
|
-
REGEX_LOG_REPLACES.each do |
|
41
|
-
msg = msg.gsub(
|
41
|
+
REGEX_LOG_REPLACES.each do |reg_ex|
|
42
|
+
msg = msg.gsub(reg_ex){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
|
42
43
|
end
|
43
44
|
end
|
44
|
-
original_formatter.call(severity,
|
45
|
+
original_formatter.call(severity, date_time, program_name, msg)
|
45
46
|
end
|
46
47
|
end
|
47
48
|
|
48
49
|
def secret?(keyword, value)
|
49
50
|
keyword = keyword.to_s if keyword.is_a?(Symbol)
|
50
51
|
# only Strings can be secrets, not booleans, or hash, arrays
|
51
|
-
keyword.is_a?(String) &&
|
52
|
+
return false unless keyword.is_a?(String) && value.is_a?(String)
|
53
|
+
# those are not secrets
|
54
|
+
return false if FALSE_POSITIVES.any?{|f|f.match?(keyword)}
|
55
|
+
# check if keyword (name) contains an element that designate it as a secret
|
56
|
+
ALL_SECRETS.any?{|kw|keyword.include?(kw)}
|
52
57
|
end
|
53
58
|
|
54
59
|
def deep_remove_secret(obj, is_name_value: false)
|
data/lib/aspera/ssh.rb
CHANGED
@@ -44,7 +44,7 @@ module Aspera
|
|
44
44
|
end
|
45
45
|
raise error_message
|
46
46
|
end
|
47
|
-
# send command to SSH channel (execute)
|
47
|
+
# send command to SSH channel (execute) cspell: disable-next-line
|
48
48
|
channel.send('cexe'.reverse, cmd){|_ch, _success|channel.send_data(input) unless input.nil?}
|
49
49
|
end
|
50
50
|
# wait for channel to finish (command exit)
|
data/lib/aspera/sync.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# cspell:words logdir
|
4
|
+
|
3
5
|
require 'aspera/command_line_builder'
|
4
6
|
require 'aspera/fasp/installation'
|
5
7
|
require 'json'
|
@@ -8,6 +10,13 @@ require 'base64'
|
|
8
10
|
module Aspera
|
9
11
|
# builds command line arg for async
|
10
12
|
class Sync
|
13
|
+
# default is push
|
14
|
+
DIRECTIONS = %i[push pull bidi].freeze
|
15
|
+
DIRECTION_TO_REQUEST_TYPE = {
|
16
|
+
push: :sync_upload,
|
17
|
+
pull: :sync_download,
|
18
|
+
bidi: :sync
|
19
|
+
}.freeze
|
11
20
|
PARAMS_VX_INSTANCE =
|
12
21
|
{
|
13
22
|
'alt_logdir' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
@@ -38,7 +47,7 @@ module Aspera
|
|
38
47
|
'cooloff' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
39
48
|
'pending_max' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
40
49
|
'scan_intensity' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
41
|
-
'cipher' => { cli: { type: :opt_with_arg}, accepted_types: :string, ts: true},
|
50
|
+
'cipher' => { cli: { type: :opt_with_arg, convert: 'Aspera::Fasp::Parameters.convert_remove_hyphen'}, accepted_types: :string, ts: true},
|
42
51
|
'transfer_threads' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
43
52
|
'preserve_time' => { cli: { type: :opt_without_arg}, ts: :preserve_times},
|
44
53
|
'preserve_access_time' => { cli: { type: :opt_without_arg}, ts: nil},
|
@@ -60,7 +69,7 @@ module Aspera
|
|
60
69
|
PARAMS_VX_KEYS = %w[instance sessions].freeze
|
61
70
|
|
62
71
|
# new API
|
63
|
-
|
72
|
+
TS_TO_PARAMS_V2 = {
|
64
73
|
'remote_host' => 'remote.host',
|
65
74
|
'remote_user' => 'remote.user',
|
66
75
|
'remote_password' => 'remote.pass',
|
@@ -74,14 +83,27 @@ module Aspera
|
|
74
83
|
|
75
84
|
ASYNC_EXECUTABLE = 'async'
|
76
85
|
|
77
|
-
private_constant :PARAMS_VX_INSTANCE, :PARAMS_VX_SESSION, :PARAMS_VX_KEYS, :ASYNC_EXECUTABLE
|
86
|
+
private_constant :PARAMS_VX_INSTANCE, :PARAMS_VX_SESSION, :PARAMS_VX_KEYS, :ASYNC_EXECUTABLE, :TS_TO_PARAMS_V2
|
87
|
+
|
88
|
+
attr_reader :env_args
|
78
89
|
|
79
|
-
|
80
|
-
|
81
|
-
|
90
|
+
# @param sync_params [Hash] sync parameters, old or new format
|
91
|
+
# @param node_sync [Object|nil]
|
92
|
+
def initialize(sync_params, node_sync)
|
93
|
+
raise StandardError, 'parameter must be Hash' unless sync_params.is_a?(Hash)
|
94
|
+
raise 'node_sync misses method transfer_spec' unless node_sync.nil? || node_sync.respond_to?(:transfer_spec)
|
95
|
+
@env_args = {
|
96
|
+
args: [],
|
97
|
+
env: {}
|
98
|
+
}
|
99
|
+
if sync_params.key?('local')
|
100
|
+
# async native JSON format (v2)
|
101
|
+
raise StandardError, 'remote must be Hash' unless sync_params['remote'].is_a?(Hash)
|
102
|
+
unless node_sync.nil?
|
103
|
+
transfer_spec = node_sync.transfer_spec(sync_params['direction'], sync_params['local']['path'], sync_params['remote']['path'])
|
82
104
|
# async native JSON format
|
83
105
|
raise StandardError, 'local must be Hash' unless sync_params['local'].is_a?(Hash)
|
84
|
-
|
106
|
+
TS_TO_PARAMS_V2.each do |ts_param, sy_path|
|
85
107
|
next unless transfer_spec.key?(ts_param)
|
86
108
|
sy_dig = sy_path.split('.')
|
87
109
|
param = sy_dig.pop
|
@@ -93,38 +115,23 @@ module Aspera
|
|
93
115
|
sync_params['remote']['connect_mode'] ||= sync_params['remote'].key?('ws_port') ? 'ws' : 'ssh'
|
94
116
|
sync_params['remote']['private_key_paths'] ||= Fasp::Installation.instance.bypass_keys if transfer_spec.key?('token')
|
95
117
|
sync_params['remote']['path'] ||= '/' if transfer_spec.dig(*%w[tags aspera node file_id])
|
96
|
-
|
118
|
+
end
|
119
|
+
@env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"]
|
120
|
+
elsif sync_params.key?('sessions')
|
121
|
+
# ascli JSON format (v1)
|
122
|
+
unless node_sync.nil?
|
97
123
|
sync_params['sessions'].each do |session|
|
98
|
-
|
99
|
-
|
100
|
-
|
124
|
+
transfer_spec = node_sync.transfer_spec(session['direction'], session['local_dir'], session['remote_dir'])
|
125
|
+
PARAMS_VX_SESSION.each do |async_param, behavior|
|
126
|
+
if behavior.key?(:ts)
|
127
|
+
tspec_param = behavior[:ts].is_a?(TrueClass) ? async_param : behavior[:ts].to_s
|
101
128
|
session[async_param] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
|
102
129
|
end
|
103
130
|
end
|
104
131
|
session['private_key_paths'] = Fasp::Installation.instance.bypass_keys if transfer_spec.key?('token')
|
105
132
|
session['remote_dir'] = '/' if transfer_spec.dig(*%w[tags aspera node file_id])
|
106
133
|
end
|
107
|
-
else
|
108
|
-
raise 'At least one of `local` or `sessions` must be present in async parameters'
|
109
134
|
end
|
110
|
-
Log.dump(:sync, sync_params)
|
111
|
-
end
|
112
|
-
end
|
113
|
-
|
114
|
-
attr_reader :env_args
|
115
|
-
|
116
|
-
def initialize(sync_params)
|
117
|
-
raise StandardError, 'parameter must be Hash' unless sync_params.is_a?(Hash)
|
118
|
-
@env_args = {
|
119
|
-
args: [],
|
120
|
-
env: {}
|
121
|
-
}
|
122
|
-
if sync_params.key?('local')
|
123
|
-
# async native JSON format
|
124
|
-
raise StandardError, 'remote must be Hash' unless sync_params['remote'].is_a?(Hash)
|
125
|
-
@env_args[:args] = "--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"
|
126
|
-
elsif sync_params.key?('sessions')
|
127
|
-
# ascli JSON format
|
128
135
|
raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
|
129
136
|
sync_params.keys.push('instance').uniq.sort.eql?(PARAMS_VX_KEYS)
|
130
137
|
raise StandardError, 'sessions key must be Array' unless sync_params['sessions'].is_a?(Array)
|
@@ -139,7 +146,7 @@ module Aspera
|
|
139
146
|
|
140
147
|
sync_params['sessions'].each do |session_params|
|
141
148
|
raise StandardError, 'sessions must contain hashes' unless session_params.is_a?(Hash)
|
142
|
-
raise StandardError, 'session must contain at
|
149
|
+
raise StandardError, 'session must contain at least name' unless session_params.key?('name')
|
143
150
|
session_builder = Aspera::CommandLineBuilder.new(session_params, PARAMS_VX_SESSION)
|
144
151
|
session_builder.process_params
|
145
152
|
session_builder.add_env_args(@env_args[:env], @env_args[:args])
|
@@ -147,6 +154,7 @@ module Aspera
|
|
147
154
|
else
|
148
155
|
raise 'At least one of `local` or `sessions` must be present in async parameters'
|
149
156
|
end
|
157
|
+
Log.dump(:sync, sync_params)
|
150
158
|
end
|
151
159
|
|
152
160
|
def start
|
@@ -160,7 +168,7 @@ module Aspera
|
|
160
168
|
else raise 'internal error: unspecified case'
|
161
169
|
end
|
162
170
|
end
|
163
|
-
end
|
171
|
+
end # end Sync
|
164
172
|
|
165
173
|
class SyncAdmin
|
166
174
|
ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
|
@@ -8,24 +8,26 @@ require 'openssl'
|
|
8
8
|
module Aspera
|
9
9
|
class WebServerSimple < WEBrick::HTTPServer
|
10
10
|
CERT_PARAMETERS = %i[key cert chain].freeze
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
11
|
+
class << self
|
12
|
+
# generates and adds self signed cert to provided webrick options
|
13
|
+
def fill_self_signed_cert(cert, key)
|
14
|
+
cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/C=FR/O=Test/OU=Test/CN=Test')
|
15
|
+
cert.not_before = Time.now
|
16
|
+
cert.not_after = Time.now + 365 * 24 * 60 * 60
|
17
|
+
cert.public_key = key.public_key
|
18
|
+
cert.serial = 0x0
|
19
|
+
cert.version = 2
|
20
|
+
ef = OpenSSL::X509::ExtensionFactory.new
|
21
|
+
ef.issuer_certificate = cert
|
22
|
+
ef.subject_certificate = cert
|
23
|
+
cert.extensions = [
|
24
|
+
ef.create_extension('basicConstraints', 'CA:TRUE', true),
|
25
|
+
ef.create_extension('subjectKeyIdentifier', 'hash')
|
26
|
+
# ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
|
27
|
+
]
|
28
|
+
cert.add_extension(ef.create_extension('authorityKeyIdentifier', 'keyid:always,issuer:always'))
|
29
|
+
cert.sign(key, OpenSSL::Digest.new('SHA256'))
|
30
|
+
end
|
29
31
|
end
|
30
32
|
|
31
33
|
# @param uri [URI]
|
@@ -66,6 +68,8 @@ module Aspera
|
|
66
68
|
end
|
67
69
|
# self signed certificate generates characters on STDERR, see create_self_signed_cert in webrick/ssl.rb
|
68
70
|
Log.capture_stderr { super(webrick_options) }
|
71
|
+
# kill -USR1 for graceful shutdown
|
72
|
+
Kernel.trap('USR1') { shutdown }
|
69
73
|
end
|
70
74
|
|
71
75
|
# log web server access ( option AccessLog )
|
data.tar.gz.sig
CHANGED
Binary file
|