aspera-cli 4.4.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2095 -1503
- data/bin/ascli +2 -1
- data/bin/asession +4 -5
- data/docs/test_env.conf +3 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +25 -25
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +17 -17
- data/lib/aspera/aoc.rb +238 -185
- data/lib/aspera/ascmd.rb +93 -83
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +142 -108
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +18 -21
- data/lib/aspera/cli/main.rb +173 -149
- data/lib/aspera/cli/manager.rb +163 -168
- data/lib/aspera/cli/plugin.rb +43 -31
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +405 -370
- data/lib/aspera/cli/plugins/ats.rb +86 -79
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +580 -362
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +201 -158
- data/lib/aspera/cli/plugins/faspex5.rb +80 -57
- data/lib/aspera/cli/plugins/node.rb +183 -166
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +35 -19
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +76 -113
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
- data/lib/aspera/fasp/agent_trsdk.rb +104 -0
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +152 -124
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +87 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -89
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +121 -0
- data/lib/aspera/keychain/macos_security.rb +90 -0
- data/lib/aspera/log.rb +55 -37
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -25
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +154 -135
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +116 -29
- data/docs/Makefile +0 -66
- data/docs/README.erb.md +0 -3973
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/secrets.rb +0 -20
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/log'
|
2
3
|
require 'aspera/rest_call_error'
|
3
4
|
require 'singleton'
|
@@ -12,10 +13,10 @@ module Aspera
|
|
12
13
|
# list of handlers
|
13
14
|
@error_handlers=[]
|
14
15
|
@log_file=nil
|
15
|
-
|
16
|
-
if !
|
16
|
+
add_handler('Type Generic') do |type,call_context|
|
17
|
+
if !call_context[:response].code.start_with?('2')
|
17
18
|
# add generic information
|
18
|
-
RestErrorAnalyzer.add_error(
|
19
|
+
RestErrorAnalyzer.add_error(call_context,type,"#{call_context[:request]['host']} #{call_context[:response].code} #{call_context[:response].message}")
|
19
20
|
end
|
20
21
|
end
|
21
22
|
end
|
@@ -23,8 +24,8 @@ module Aspera
|
|
23
24
|
# Use this method to analyze a EST result and raise an exception
|
24
25
|
# Analyzes REST call response and raises a RestCallError exception
|
25
26
|
# if HTTP result code is not 2XX
|
26
|
-
def
|
27
|
-
|
27
|
+
def raise_on_error(req,res)
|
28
|
+
call_context={
|
28
29
|
messages: [],
|
29
30
|
request: req,
|
30
31
|
response: res[:http],
|
@@ -36,21 +37,19 @@ module Aspera
|
|
36
37
|
@error_handlers.each do |handler|
|
37
38
|
begin
|
38
39
|
#Log.log.debug("test exception: #{handler[:name]}")
|
39
|
-
handler[:block].call(handler[:name],
|
40
|
-
rescue => e
|
40
|
+
handler[:block].call(handler[:name],call_context)
|
41
|
+
rescue StandardError => e
|
41
42
|
Log.log.error("ERROR in handler:\n#{e.message}\n#{e.backtrace}")
|
42
43
|
end
|
43
44
|
end
|
44
|
-
unless
|
45
|
-
raise RestCallError.new(context[:request],context[:response],context[:messages].join("\n"))
|
46
|
-
end
|
45
|
+
raise RestCallError.new(call_context[:request],call_context[:response],call_context[:messages].join("\n")) unless call_context[:messages].empty?
|
47
46
|
end
|
48
47
|
|
49
48
|
# add a new error handler (done at application initialisation)
|
50
49
|
# @param name : name of error handler (for logs)
|
51
|
-
# @param block : processing of response: takes two parameters: name,
|
50
|
+
# @param block : processing of response: takes two parameters: name, call_context
|
52
51
|
# name is the one provided here
|
53
|
-
#
|
52
|
+
# call_context is built in method raise_on_error
|
54
53
|
def add_handler(name,&block)
|
55
54
|
@error_handlers.unshift({name: name, block: block})
|
56
55
|
end
|
@@ -59,21 +58,21 @@ module Aspera
|
|
59
58
|
# check that key exists and is string under specified path (hash)
|
60
59
|
# adds other keys as secondary information
|
61
60
|
def add_simple_handler(name,*args)
|
62
|
-
add_handler(name) do |type,
|
61
|
+
add_handler(name) do |type,call_context|
|
63
62
|
# need to clone because we modify and same array is used subsequently
|
64
63
|
path=args.clone
|
65
64
|
#Log.log.debug("path=#{path}")
|
66
65
|
# if last in path is boolean it tells if the error is only with http error code or always
|
67
66
|
always=[true, false].include?(path.last) ? path.pop : false
|
68
|
-
if
|
67
|
+
if call_context[:data].is_a?(Hash) && (!call_context[:response].code.start_with?('2') || always)
|
69
68
|
msg_key=path.pop
|
70
69
|
# dig and find sub entry corresponding to path in deep hash
|
71
|
-
error_struct=path.inject(
|
72
|
-
if error_struct.is_a?(Hash)
|
73
|
-
RestErrorAnalyzer.add_error(
|
70
|
+
error_struct=path.inject(call_context[:data]) { |subhash, key| subhash.respond_to?(:keys) ? subhash[key] : nil }
|
71
|
+
if error_struct.is_a?(Hash) && error_struct[msg_key].is_a?(String)
|
72
|
+
RestErrorAnalyzer.add_error(call_context,type,error_struct[msg_key])
|
74
73
|
error_struct.each do |k,v|
|
75
74
|
next if k.eql?(msg_key)
|
76
|
-
RestErrorAnalyzer.add_error(
|
75
|
+
RestErrorAnalyzer.add_error(call_context,"#{type}(sub)","#{k}: #{v}") if [String,Integer].include?(v.class)
|
77
76
|
end
|
78
77
|
end
|
79
78
|
end
|
@@ -82,17 +81,16 @@ module Aspera
|
|
82
81
|
|
83
82
|
# used by handler to add an error description to list of errors
|
84
83
|
# for logging and tracing : collect error descriptions (create file to activate)
|
85
|
-
# @param
|
84
|
+
# @param call_context a Hash containing the result call_context, provided to handler
|
86
85
|
# @param type a string describing type of exception, for logging purpose
|
87
86
|
# @param msg one error message to add to list
|
88
|
-
def self.add_error(
|
89
|
-
|
87
|
+
def self.add_error(call_context,type,msg)
|
88
|
+
call_context[:messages].push(msg)
|
90
89
|
logfile=instance.log_file
|
91
90
|
# log error for further analysis (file must exist to activate)
|
92
|
-
if
|
93
|
-
|
94
|
-
|
95
|
-
end
|
91
|
+
return if logfile.nil? || !File.exist?(logfile)
|
92
|
+
File.open(logfile,'a+') do |f|
|
93
|
+
f.write("\n=#{type}=====\n#{call_context[:request].method} #{call_context[:request].path}\n#{call_context[:response].code}\n#{JSON.generate(call_context[:data])}\n#{call_context[:messages].join("\n")}")
|
96
94
|
end
|
97
95
|
end
|
98
96
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/rest_error_analyzer'
|
2
3
|
require 'aspera/log'
|
3
4
|
|
@@ -5,8 +6,8 @@ module Aspera
|
|
5
6
|
# REST error handlers for various Aspera REST APIs
|
6
7
|
class RestErrorsAspera
|
7
8
|
# handlers should probably be defined by plugins for modularity
|
8
|
-
def self.
|
9
|
-
Log.log.debug(
|
9
|
+
def self.register_handlers
|
10
|
+
Log.log.debug('registering Aspera REST error handlers')
|
10
11
|
# Faspex 4: both user_message and internal_message, and code 200
|
11
12
|
# example: missing meta data on package creation
|
12
13
|
RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
|
@@ -16,23 +17,23 @@ module Aspera
|
|
16
17
|
RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
|
17
18
|
RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
|
18
19
|
RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
|
19
|
-
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,
|
20
|
-
if
|
21
|
-
|
22
|
-
RestErrorAnalyzer.add_error(
|
20
|
+
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,call_context|
|
21
|
+
if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
|
22
|
+
call_context[:data]['errors'].each do |k,v|
|
23
|
+
RestErrorAnalyzer.add_error(call_context,name,"#{k}: #{v}")
|
23
24
|
end
|
24
25
|
end
|
25
26
|
end
|
26
27
|
# call to upload_setup and download_setup of node api
|
27
|
-
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,
|
28
|
-
if
|
29
|
-
d_t_s=
|
28
|
+
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
|
29
|
+
if call_context[:data].is_a?(Hash)
|
30
|
+
d_t_s=call_context[:data]['transfer_specs']
|
30
31
|
if d_t_s.is_a?(Array)
|
31
32
|
d_t_s.each do |res|
|
32
33
|
#r_err=res['transfer_spec']['error']
|
33
34
|
r_err=res['error']
|
34
35
|
if r_err.is_a?(Hash)
|
35
|
-
RestErrorAnalyzer.add_error(
|
36
|
+
RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
36
37
|
end
|
37
38
|
end
|
38
39
|
end
|
@@ -40,14 +41,14 @@ module Aspera
|
|
40
41
|
end
|
41
42
|
RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
|
42
43
|
RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
|
43
|
-
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,
|
44
|
-
if
|
45
|
-
d_t_s=
|
44
|
+
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
|
45
|
+
if call_context[:data].is_a?(Hash)
|
46
|
+
d_t_s=call_context[:data]['errors']
|
46
47
|
if d_t_s.is_a?(Array)
|
47
48
|
d_t_s.each do |res|
|
48
49
|
r_err=res['message']
|
49
50
|
if r_err.is_a?(String)
|
50
|
-
RestErrorAnalyzer.add_error(
|
51
|
+
RestErrorAnalyzer.add_error(call_context,type,r_err)
|
51
52
|
end
|
52
53
|
end
|
53
54
|
end
|
data/lib/aspera/ssh.rb
CHANGED
@@ -1,9 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'net/ssh'
|
2
3
|
|
3
4
|
# Hack: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually hurts
|
4
5
|
begin
|
5
|
-
module Net;
|
6
|
-
rescue
|
6
|
+
module Net;module SSH;module Authentication;class Session;private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa];end;end;end;end;end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier
|
7
|
+
rescue StandardError
|
8
|
+
# ignore errors
|
7
9
|
end
|
8
10
|
|
9
11
|
module Aspera
|
@@ -23,31 +25,31 @@ module Aspera
|
|
23
25
|
def execute(cmd,input=nil)
|
24
26
|
if cmd.is_a?(Array)
|
25
27
|
# concatenate arguments, enclose in double quotes
|
26
|
-
cmd=cmd.map{|v
|
28
|
+
cmd=cmd.map{|v|%Q("#{v}")}.join(' ')
|
27
29
|
end
|
28
30
|
Log.log.debug("cmd=#{cmd}")
|
29
|
-
response =
|
31
|
+
response = []
|
30
32
|
Net::SSH.start(@host, @username, @ssh_options) do |session|
|
31
33
|
ssh_channel=session.open_channel do |channel|
|
32
34
|
# prepare stdout processing
|
33
|
-
channel.on_data{|
|
35
|
+
channel.on_data{|_chan,data|response.push(data)}
|
34
36
|
# prepare stderr processing, stderr if type = 1
|
35
|
-
channel.on_extended_data do |
|
37
|
+
channel.on_extended_data do |_chan, _type, data|
|
36
38
|
errormsg="#{cmd}: [#{data.chomp}]"
|
37
39
|
# Happens when windows user hasn't logged in and created home account.
|
38
|
-
if data.include?(
|
39
|
-
errormsg
|
40
|
+
if data.include?('Could not chdir to home directory')
|
41
|
+
errormsg+="\nHint: home not created in Windows?"
|
40
42
|
end
|
41
43
|
raise errormsg
|
42
44
|
end
|
43
|
-
channel.exec(cmd){|
|
45
|
+
channel.exec(cmd){|_ch,_success|channel.send_data(input) unless input.nil?}
|
44
46
|
end
|
45
47
|
# wait for channel to finish (command exit)
|
46
48
|
ssh_channel.wait
|
47
49
|
# main ssh session loop
|
48
50
|
session.loop
|
49
51
|
end # session
|
50
|
-
return response
|
52
|
+
return response.join
|
51
53
|
end
|
52
54
|
end
|
53
55
|
end
|
data/lib/aspera/sync.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/command_line_builder'
|
2
3
|
|
3
4
|
module Aspera
|
@@ -5,43 +6,43 @@ module Aspera
|
|
5
6
|
class Sync
|
6
7
|
INSTANCE_PARAMS=
|
7
8
|
{
|
8
|
-
'alt_logdir' => { :
|
9
|
-
'watchd' => { :
|
10
|
-
'apply_local_docroot' => { :
|
11
|
-
'quiet' => { :
|
9
|
+
'alt_logdir' => { cltype: :opt_with_arg, accepted_types: :string},
|
10
|
+
'watchd' => { cltype: :opt_with_arg, accepted_types: :string},
|
11
|
+
'apply_local_docroot' => { cltype: :opt_without_arg},
|
12
|
+
'quiet' => { cltype: :opt_without_arg}
|
12
13
|
}
|
13
14
|
SESSION_PARAMS=
|
14
15
|
{
|
15
|
-
'name' => { :
|
16
|
-
'local_dir' => { :
|
17
|
-
'remote_dir' => { :
|
18
|
-
'local_db_dir' => { :
|
19
|
-
'remote_db_dir' => { :
|
20
|
-
'host' => { :
|
21
|
-
'user' => { :
|
22
|
-
'private_key_path' => { :
|
23
|
-
'direction' => { :
|
24
|
-
'checksum' => { :
|
25
|
-
'tcp_port' => { :
|
26
|
-
'rate_policy' => { :
|
27
|
-
'target_rate' => { :
|
28
|
-
'cooloff' => { :
|
29
|
-
'pending_max' => { :
|
30
|
-
'scan_intensity' => { :
|
31
|
-
'cipher' => { :
|
32
|
-
'transfer_threads' => { :
|
33
|
-
'preserve_time' => { :
|
34
|
-
'preserve_access_time' => { :
|
35
|
-
'preserve_modification_time' => { :
|
36
|
-
'preserve_uid' => { :
|
37
|
-
'preserve_gid' => { :
|
38
|
-
'create_dir' => { :
|
39
|
-
'reset' => { :
|
16
|
+
'name' => { cltype: :opt_with_arg, accepted_types: :string},
|
17
|
+
'local_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
18
|
+
'remote_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
19
|
+
'local_db_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
20
|
+
'remote_db_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
21
|
+
'host' => { cltype: :opt_with_arg, accepted_types: :string},
|
22
|
+
'user' => { cltype: :opt_with_arg, accepted_types: :string},
|
23
|
+
'private_key_path' => { cltype: :opt_with_arg, accepted_types: :string},
|
24
|
+
'direction' => { cltype: :opt_with_arg, accepted_types: :string},
|
25
|
+
'checksum' => { cltype: :opt_with_arg, accepted_types: :string},
|
26
|
+
'tcp_port' => { cltype: :opt_with_arg, accepted_types: :int},
|
27
|
+
'rate_policy' => { cltype: :opt_with_arg, accepted_types: :string},
|
28
|
+
'target_rate' => { cltype: :opt_with_arg, accepted_types: :string},
|
29
|
+
'cooloff' => { cltype: :opt_with_arg, accepted_types: :int},
|
30
|
+
'pending_max' => { cltype: :opt_with_arg, accepted_types: :int},
|
31
|
+
'scan_intensity' => { cltype: :opt_with_arg, accepted_types: :string},
|
32
|
+
'cipher' => { cltype: :opt_with_arg, accepted_types: :string},
|
33
|
+
'transfer_threads' => { cltype: :opt_with_arg, accepted_types: :int},
|
34
|
+
'preserve_time' => { cltype: :opt_without_arg},
|
35
|
+
'preserve_access_time' => { cltype: :opt_without_arg},
|
36
|
+
'preserve_modification_time' => { cltype: :opt_without_arg},
|
37
|
+
'preserve_uid' => { cltype: :opt_without_arg},
|
38
|
+
'preserve_gid' => { cltype: :opt_without_arg},
|
39
|
+
'create_dir' => { cltype: :opt_without_arg},
|
40
|
+
'reset' => { cltype: :opt_without_arg},
|
40
41
|
# note: only one env var, but multiple sessions... may be a problem
|
41
|
-
'remote_password' => { :
|
42
|
-
'cookie' => { :
|
43
|
-
'token' => { :
|
44
|
-
'license' => { :
|
42
|
+
'remote_password' => { cltype: :envvar, clvarname: 'ASPERA_SCP_PASS'},
|
43
|
+
'cookie' => { cltype: :envvar, clvarname: 'ASPERA_SCP_COOKIE'},
|
44
|
+
'token' => { cltype: :envvar, clvarname: 'ASPERA_SCP_TOKEN'},
|
45
|
+
'license' => { cltype: :envvar, clvarname: 'ASPERA_SCP_LICENSE'}
|
45
46
|
}
|
46
47
|
|
47
48
|
Aspera::CommandLineBuilder.normalize_description(INSTANCE_PARAMS)
|
@@ -56,26 +57,26 @@ module Aspera
|
|
56
57
|
MANDATORY_KEYS=['instance','sessions']
|
57
58
|
|
58
59
|
def compute_args
|
59
|
-
raise StandardError,
|
60
|
+
raise StandardError,'parameter must be Hash' unless @sync_params.is_a?(Hash)
|
60
61
|
raise StandardError,"parameter hash must have at least 'sessions', and optionally 'instance' keys." unless @sync_params.keys.push('instance').uniq.sort.eql?(MANDATORY_KEYS)
|
61
|
-
raise StandardError,
|
62
|
-
raise StandardError,
|
62
|
+
raise StandardError,'sessions key must be Array' unless @sync_params['sessions'].is_a?(Array)
|
63
|
+
raise StandardError,'sessions key must has at least one element (hash)' unless @sync_params['sessions'].first.is_a?(Hash)
|
63
64
|
|
64
65
|
env_args={
|
65
|
-
:
|
66
|
-
:
|
66
|
+
args: [],
|
67
|
+
env: {}
|
67
68
|
}
|
68
69
|
|
69
70
|
if @sync_params.has_key?('instance')
|
70
|
-
raise StandardError,
|
71
|
+
raise StandardError,'instance key must be hash' unless @sync_params['instance'].is_a?(Hash)
|
71
72
|
instance_builder=CommandLineBuilder.new(@sync_params['instance'],INSTANCE_PARAMS)
|
72
73
|
instance_builder.process_params
|
73
74
|
instance_builder.add_env_args(env_args[:env],env_args[:args])
|
74
75
|
end
|
75
76
|
|
76
77
|
@sync_params['sessions'].each do |session_params|
|
77
|
-
raise StandardError,
|
78
|
-
raise StandardError,
|
78
|
+
raise StandardError,'sessions must contain hashes' unless session_params.is_a?(Hash)
|
79
|
+
raise StandardError,'session must contain at leat name' unless session_params.has_key?('name')
|
79
80
|
session_builder=CommandLineBuilder.new(session_params,SESSION_PARAMS)
|
80
81
|
session_builder.process_params
|
81
82
|
session_builder.add_env_args(env_args[:env],env_args[:args])
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'singleton'
|
2
3
|
require 'fileutils'
|
3
4
|
require 'etc'
|
@@ -6,14 +7,14 @@ module Aspera
|
|
6
7
|
# create a temp file name for a given folder
|
7
8
|
# files can be deleted on process exit by calling cleanup
|
8
9
|
class TempFileManager
|
9
|
-
SEC_IN_DAY=
|
10
|
+
SEC_IN_DAY = 86_400
|
10
11
|
# assume no transfer last longer than this
|
11
12
|
# (garbage collect file list which were not deleted after transfer)
|
12
|
-
FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
|
13
|
-
private_constant :SEC_IN_DAY
|
13
|
+
FILE_LIST_AGE_MAX_SEC = 5 * SEC_IN_DAY
|
14
|
+
private_constant :SEC_IN_DAY, :FILE_LIST_AGE_MAX_SEC
|
14
15
|
include Singleton
|
15
16
|
def initialize
|
16
|
-
@created_files=[]
|
17
|
+
@created_files = []
|
17
18
|
end
|
18
19
|
|
19
20
|
# call this on process exit
|
@@ -21,36 +22,39 @@ module Aspera
|
|
21
22
|
@created_files.each do |filepath|
|
22
23
|
File.delete(filepath) if File.file?(filepath)
|
23
24
|
end
|
24
|
-
@created_files=[]
|
25
|
+
@created_files = []
|
25
26
|
end
|
26
27
|
|
27
28
|
# ensure that provided folder exists, or create it, generate a unique filename
|
28
29
|
# @return path to that unique file
|
29
|
-
def new_file_path_in_folder(temp_folder,add_base='')
|
30
|
+
def new_file_path_in_folder(temp_folder, add_base = '')
|
30
31
|
FileUtils.mkdir_p(temp_folder) unless Dir.exist?(temp_folder)
|
31
|
-
new_file=File.join(temp_folder,add_base+SecureRandom.uuid)
|
32
|
+
new_file = File.join(temp_folder, add_base + SecureRandom.uuid)
|
32
33
|
@created_files.push(new_file)
|
33
|
-
|
34
|
+
new_file
|
34
35
|
end
|
35
36
|
|
36
37
|
# same as above but in global temp folder
|
37
38
|
def new_file_path_global(base_name)
|
38
|
-
username =
|
39
|
-
|
39
|
+
username = begin
|
40
|
+
Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
|
41
|
+
rescue StandardError
|
42
|
+
'unknown_user'
|
43
|
+
end
|
44
|
+
new_file_path_in_folder(Etc.systmpdir, base_name + '_' + username + '_')
|
40
45
|
end
|
41
46
|
|
42
47
|
def cleanup_expired(temp_folder)
|
43
48
|
# garbage collect undeleted files
|
44
49
|
Dir.entries(temp_folder).each do |name|
|
45
|
-
file_path=File.join(temp_folder,name)
|
46
|
-
age_sec=(Time.now - File.stat(file_path).mtime).to_i
|
50
|
+
file_path = File.join(temp_folder, name)
|
51
|
+
age_sec = (Time.now - File.stat(file_path).mtime).to_i
|
47
52
|
# check age of file, delete too old
|
48
|
-
if File.file?(file_path)
|
53
|
+
if File.file?(file_path) && (age_sec > FILE_LIST_AGE_MAX_SEC)
|
49
54
|
Log.log.debug("garbage collecting #{name}")
|
50
55
|
File.delete(file_path)
|
51
56
|
end
|
52
57
|
end
|
53
|
-
|
54
58
|
end
|
55
59
|
end
|
56
60
|
end
|
data/lib/aspera/timer_limiter.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module Aspera
|
2
3
|
# used to throttle logs
|
3
4
|
class TimerLimiter
|
@@ -12,7 +13,7 @@ module Aspera
|
|
12
13
|
old_time=@last_time
|
13
14
|
@last_time=Time.now.to_f
|
14
15
|
@count+=1
|
15
|
-
if old_time.nil?
|
16
|
+
if old_time.nil? || ((@last_time-old_time)>@delay)
|
16
17
|
@count=0
|
17
18
|
return true
|
18
19
|
end
|
data/lib/aspera/uri_reader.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'uri'
|
2
3
|
require 'net/http'
|
3
4
|
require 'net/https'
|
@@ -7,16 +8,17 @@ module Aspera
|
|
7
8
|
# read some content from some URI, support file: , http: and https: schemes
|
8
9
|
def self.read(proxy_pac_uri)
|
9
10
|
proxy_uri=URI.parse(proxy_pac_uri)
|
10
|
-
|
11
|
+
case proxy_uri.scheme
|
12
|
+
when 'http'
|
11
13
|
return Net::HTTP.start(proxy_uri.host, proxy_uri.port){|http|http.get(proxy_uri.path)}.body
|
12
|
-
|
14
|
+
when 'https'
|
13
15
|
return Net::HTTPS.start(proxy_uri.host, proxy_uri.port){|http|http.get(proxy_uri.path)}.body
|
14
|
-
|
16
|
+
when 'file'
|
15
17
|
local_file_path=proxy_uri.path
|
16
|
-
raise
|
18
|
+
raise 'URL shall have a path, check syntax' if local_file_path.nil?
|
17
19
|
local_file_path=File.expand_path(local_file_path.gsub(/^\//,'')) if local_file_path.match(/^\/(~|.|..)\//)
|
18
20
|
return File.read(local_file_path)
|
19
|
-
|
21
|
+
when ''
|
20
22
|
return File.read(proxy_uri)
|
21
23
|
end
|
22
24
|
raise "no scheme: [#{proxy_uri.scheme}] for [#{proxy_pac_uri}]"
|