aspera-cli 4.6.0 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/rest_error_analyzer'
|
|
2
3
|
require 'aspera/log'
|
|
3
4
|
|
|
@@ -5,8 +6,8 @@ module Aspera
|
|
|
5
6
|
# REST error handlers for various Aspera REST APIs
|
|
6
7
|
class RestErrorsAspera
|
|
7
8
|
# handlers should probably be defined by plugins for modularity
|
|
8
|
-
def self.
|
|
9
|
-
Log.log.debug(
|
|
9
|
+
def self.register_handlers
|
|
10
|
+
Log.log.debug('registering Aspera REST error handlers')
|
|
10
11
|
# Faspex 4: both user_message and internal_message, and code 200
|
|
11
12
|
# example: missing meta data on package creation
|
|
12
13
|
RestErrorAnalyzer.instance.add_simple_handler('Type 1: error:user_message','error','user_message',true)
|
|
@@ -16,23 +17,23 @@ module Aspera
|
|
|
16
17
|
RestErrorAnalyzer.instance.add_simple_handler('AoC Automation','error')
|
|
17
18
|
RestErrorAnalyzer.instance.add_simple_handler('Type 5','error_description')
|
|
18
19
|
RestErrorAnalyzer.instance.add_simple_handler('Type 6','message')
|
|
19
|
-
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,
|
|
20
|
-
if
|
|
21
|
-
|
|
22
|
-
RestErrorAnalyzer.add_error(
|
|
20
|
+
RestErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |name,call_context|
|
|
21
|
+
if call_context[:data].is_a?(Hash) && call_context[:data]['errors'].is_a?(Hash)
|
|
22
|
+
call_context[:data]['errors'].each do |k,v|
|
|
23
|
+
RestErrorAnalyzer.add_error(call_context,name,"#{k}: #{v}")
|
|
23
24
|
end
|
|
24
25
|
end
|
|
25
26
|
end
|
|
26
27
|
# call to upload_setup and download_setup of node api
|
|
27
|
-
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,
|
|
28
|
-
if
|
|
29
|
-
d_t_s=
|
|
28
|
+
RestErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type,call_context|
|
|
29
|
+
if call_context[:data].is_a?(Hash)
|
|
30
|
+
d_t_s=call_context[:data]['transfer_specs']
|
|
30
31
|
if d_t_s.is_a?(Array)
|
|
31
32
|
d_t_s.each do |res|
|
|
32
33
|
#r_err=res['transfer_spec']['error']
|
|
33
34
|
r_err=res['error']
|
|
34
35
|
if r_err.is_a?(Hash)
|
|
35
|
-
RestErrorAnalyzer.add_error(
|
|
36
|
+
RestErrorAnalyzer.add_error(call_context,type,"#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
|
|
36
37
|
end
|
|
37
38
|
end
|
|
38
39
|
end
|
|
@@ -40,14 +41,14 @@ module Aspera
|
|
|
40
41
|
end
|
|
41
42
|
RestErrorAnalyzer.instance.add_simple_handler('T9:IBM cloud IAM','errorMessage')
|
|
42
43
|
RestErrorAnalyzer.instance.add_simple_handler('T10:faspex v4','user_message')
|
|
43
|
-
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,
|
|
44
|
-
if
|
|
45
|
-
d_t_s=
|
|
44
|
+
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type,call_context|
|
|
45
|
+
if call_context[:data].is_a?(Hash)
|
|
46
|
+
d_t_s=call_context[:data]['errors']
|
|
46
47
|
if d_t_s.is_a?(Array)
|
|
47
48
|
d_t_s.each do |res|
|
|
48
49
|
r_err=res['message']
|
|
49
50
|
if r_err.is_a?(String)
|
|
50
|
-
RestErrorAnalyzer.add_error(
|
|
51
|
+
RestErrorAnalyzer.add_error(call_context,type,r_err)
|
|
51
52
|
end
|
|
52
53
|
end
|
|
53
54
|
end
|
data/lib/aspera/ssh.rb
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'net/ssh'
|
|
2
3
|
|
|
3
4
|
# Hack: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually hurts
|
|
4
5
|
begin
|
|
5
|
-
module Net;
|
|
6
|
-
rescue
|
|
6
|
+
module Net;module SSH;module Authentication;class Session;private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa];end;end;end;end;end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier
|
|
7
|
+
rescue StandardError
|
|
8
|
+
# ignore errors
|
|
7
9
|
end
|
|
8
10
|
|
|
9
11
|
module Aspera
|
|
@@ -23,31 +25,31 @@ module Aspera
|
|
|
23
25
|
def execute(cmd,input=nil)
|
|
24
26
|
if cmd.is_a?(Array)
|
|
25
27
|
# concatenate arguments, enclose in double quotes
|
|
26
|
-
cmd=cmd.map{|v
|
|
28
|
+
cmd=cmd.map{|v|%Q("#{v}")}.join(' ')
|
|
27
29
|
end
|
|
28
30
|
Log.log.debug("cmd=#{cmd}")
|
|
29
|
-
response =
|
|
31
|
+
response = []
|
|
30
32
|
Net::SSH.start(@host, @username, @ssh_options) do |session|
|
|
31
33
|
ssh_channel=session.open_channel do |channel|
|
|
32
34
|
# prepare stdout processing
|
|
33
|
-
channel.on_data{|
|
|
35
|
+
channel.on_data{|_chan,data|response.push(data)}
|
|
34
36
|
# prepare stderr processing, stderr if type = 1
|
|
35
|
-
channel.on_extended_data do |
|
|
37
|
+
channel.on_extended_data do |_chan, _type, data|
|
|
36
38
|
errormsg="#{cmd}: [#{data.chomp}]"
|
|
37
39
|
# Happens when windows user hasn't logged in and created home account.
|
|
38
|
-
if data.include?(
|
|
39
|
-
errormsg
|
|
40
|
+
if data.include?('Could not chdir to home directory')
|
|
41
|
+
errormsg+="\nHint: home not created in Windows?"
|
|
40
42
|
end
|
|
41
43
|
raise errormsg
|
|
42
44
|
end
|
|
43
|
-
channel.exec(cmd){|
|
|
45
|
+
channel.exec(cmd){|_ch,_success|channel.send_data(input) unless input.nil?}
|
|
44
46
|
end
|
|
45
47
|
# wait for channel to finish (command exit)
|
|
46
48
|
ssh_channel.wait
|
|
47
49
|
# main ssh session loop
|
|
48
50
|
session.loop
|
|
49
51
|
end # session
|
|
50
|
-
return response
|
|
52
|
+
return response.join
|
|
51
53
|
end
|
|
52
54
|
end
|
|
53
55
|
end
|
data/lib/aspera/sync.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/command_line_builder'
|
|
2
3
|
|
|
3
4
|
module Aspera
|
|
@@ -5,43 +6,43 @@ module Aspera
|
|
|
5
6
|
class Sync
|
|
6
7
|
INSTANCE_PARAMS=
|
|
7
8
|
{
|
|
8
|
-
'alt_logdir' => { :
|
|
9
|
-
'watchd' => { :
|
|
10
|
-
'apply_local_docroot' => { :
|
|
11
|
-
'quiet' => { :
|
|
9
|
+
'alt_logdir' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
10
|
+
'watchd' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
11
|
+
'apply_local_docroot' => { cltype: :opt_without_arg},
|
|
12
|
+
'quiet' => { cltype: :opt_without_arg}
|
|
12
13
|
}
|
|
13
14
|
SESSION_PARAMS=
|
|
14
15
|
{
|
|
15
|
-
'name' => { :
|
|
16
|
-
'local_dir' => { :
|
|
17
|
-
'remote_dir' => { :
|
|
18
|
-
'local_db_dir' => { :
|
|
19
|
-
'remote_db_dir' => { :
|
|
20
|
-
'host' => { :
|
|
21
|
-
'user' => { :
|
|
22
|
-
'private_key_path' => { :
|
|
23
|
-
'direction' => { :
|
|
24
|
-
'checksum' => { :
|
|
25
|
-
'tcp_port' => { :
|
|
26
|
-
'rate_policy' => { :
|
|
27
|
-
'target_rate' => { :
|
|
28
|
-
'cooloff' => { :
|
|
29
|
-
'pending_max' => { :
|
|
30
|
-
'scan_intensity' => { :
|
|
31
|
-
'cipher' => { :
|
|
32
|
-
'transfer_threads' => { :
|
|
33
|
-
'preserve_time' => { :
|
|
34
|
-
'preserve_access_time' => { :
|
|
35
|
-
'preserve_modification_time' => { :
|
|
36
|
-
'preserve_uid' => { :
|
|
37
|
-
'preserve_gid' => { :
|
|
38
|
-
'create_dir' => { :
|
|
39
|
-
'reset' => { :
|
|
16
|
+
'name' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
17
|
+
'local_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
18
|
+
'remote_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
19
|
+
'local_db_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
20
|
+
'remote_db_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
21
|
+
'host' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
22
|
+
'user' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
23
|
+
'private_key_path' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
24
|
+
'direction' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
25
|
+
'checksum' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
26
|
+
'tcp_port' => { cltype: :opt_with_arg, accepted_types: :int},
|
|
27
|
+
'rate_policy' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
28
|
+
'target_rate' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
29
|
+
'cooloff' => { cltype: :opt_with_arg, accepted_types: :int},
|
|
30
|
+
'pending_max' => { cltype: :opt_with_arg, accepted_types: :int},
|
|
31
|
+
'scan_intensity' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
32
|
+
'cipher' => { cltype: :opt_with_arg, accepted_types: :string},
|
|
33
|
+
'transfer_threads' => { cltype: :opt_with_arg, accepted_types: :int},
|
|
34
|
+
'preserve_time' => { cltype: :opt_without_arg},
|
|
35
|
+
'preserve_access_time' => { cltype: :opt_without_arg},
|
|
36
|
+
'preserve_modification_time' => { cltype: :opt_without_arg},
|
|
37
|
+
'preserve_uid' => { cltype: :opt_without_arg},
|
|
38
|
+
'preserve_gid' => { cltype: :opt_without_arg},
|
|
39
|
+
'create_dir' => { cltype: :opt_without_arg},
|
|
40
|
+
'reset' => { cltype: :opt_without_arg},
|
|
40
41
|
# note: only one env var, but multiple sessions... may be a problem
|
|
41
|
-
'remote_password' => { :
|
|
42
|
-
'cookie' => { :
|
|
43
|
-
'token' => { :
|
|
44
|
-
'license' => { :
|
|
42
|
+
'remote_password' => { cltype: :envvar, clvarname: 'ASPERA_SCP_PASS'},
|
|
43
|
+
'cookie' => { cltype: :envvar, clvarname: 'ASPERA_SCP_COOKIE'},
|
|
44
|
+
'token' => { cltype: :envvar, clvarname: 'ASPERA_SCP_TOKEN'},
|
|
45
|
+
'license' => { cltype: :envvar, clvarname: 'ASPERA_SCP_LICENSE'}
|
|
45
46
|
}
|
|
46
47
|
|
|
47
48
|
Aspera::CommandLineBuilder.normalize_description(INSTANCE_PARAMS)
|
|
@@ -56,26 +57,26 @@ module Aspera
|
|
|
56
57
|
MANDATORY_KEYS=['instance','sessions']
|
|
57
58
|
|
|
58
59
|
def compute_args
|
|
59
|
-
raise StandardError,
|
|
60
|
+
raise StandardError,'parameter must be Hash' unless @sync_params.is_a?(Hash)
|
|
60
61
|
raise StandardError,"parameter hash must have at least 'sessions', and optionally 'instance' keys." unless @sync_params.keys.push('instance').uniq.sort.eql?(MANDATORY_KEYS)
|
|
61
|
-
raise StandardError,
|
|
62
|
-
raise StandardError,
|
|
62
|
+
raise StandardError,'sessions key must be Array' unless @sync_params['sessions'].is_a?(Array)
|
|
63
|
+
raise StandardError,'sessions key must has at least one element (hash)' unless @sync_params['sessions'].first.is_a?(Hash)
|
|
63
64
|
|
|
64
65
|
env_args={
|
|
65
|
-
:
|
|
66
|
-
:
|
|
66
|
+
args: [],
|
|
67
|
+
env: {}
|
|
67
68
|
}
|
|
68
69
|
|
|
69
70
|
if @sync_params.has_key?('instance')
|
|
70
|
-
raise StandardError,
|
|
71
|
+
raise StandardError,'instance key must be hash' unless @sync_params['instance'].is_a?(Hash)
|
|
71
72
|
instance_builder=CommandLineBuilder.new(@sync_params['instance'],INSTANCE_PARAMS)
|
|
72
73
|
instance_builder.process_params
|
|
73
74
|
instance_builder.add_env_args(env_args[:env],env_args[:args])
|
|
74
75
|
end
|
|
75
76
|
|
|
76
77
|
@sync_params['sessions'].each do |session_params|
|
|
77
|
-
raise StandardError,
|
|
78
|
-
raise StandardError,
|
|
78
|
+
raise StandardError,'sessions must contain hashes' unless session_params.is_a?(Hash)
|
|
79
|
+
raise StandardError,'session must contain at leat name' unless session_params.has_key?('name')
|
|
79
80
|
session_builder=CommandLineBuilder.new(session_params,SESSION_PARAMS)
|
|
80
81
|
session_builder.process_params
|
|
81
82
|
session_builder.add_env_args(env_args[:env],env_args[:args])
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'singleton'
|
|
2
3
|
require 'fileutils'
|
|
3
4
|
require 'etc'
|
|
@@ -6,14 +7,14 @@ module Aspera
|
|
|
6
7
|
# create a temp file name for a given folder
|
|
7
8
|
# files can be deleted on process exit by calling cleanup
|
|
8
9
|
class TempFileManager
|
|
9
|
-
SEC_IN_DAY=
|
|
10
|
+
SEC_IN_DAY = 86_400
|
|
10
11
|
# assume no transfer last longer than this
|
|
11
12
|
# (garbage collect file list which were not deleted after transfer)
|
|
12
|
-
FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
|
|
13
|
-
private_constant :SEC_IN_DAY
|
|
13
|
+
FILE_LIST_AGE_MAX_SEC = 5 * SEC_IN_DAY
|
|
14
|
+
private_constant :SEC_IN_DAY, :FILE_LIST_AGE_MAX_SEC
|
|
14
15
|
include Singleton
|
|
15
16
|
def initialize
|
|
16
|
-
@created_files=[]
|
|
17
|
+
@created_files = []
|
|
17
18
|
end
|
|
18
19
|
|
|
19
20
|
# call this on process exit
|
|
@@ -21,36 +22,39 @@ module Aspera
|
|
|
21
22
|
@created_files.each do |filepath|
|
|
22
23
|
File.delete(filepath) if File.file?(filepath)
|
|
23
24
|
end
|
|
24
|
-
@created_files=[]
|
|
25
|
+
@created_files = []
|
|
25
26
|
end
|
|
26
27
|
|
|
27
28
|
# ensure that provided folder exists, or create it, generate a unique filename
|
|
28
29
|
# @return path to that unique file
|
|
29
|
-
def new_file_path_in_folder(temp_folder,add_base='')
|
|
30
|
+
def new_file_path_in_folder(temp_folder, add_base = '')
|
|
30
31
|
FileUtils.mkdir_p(temp_folder) unless Dir.exist?(temp_folder)
|
|
31
|
-
new_file=File.join(temp_folder,add_base+SecureRandom.uuid)
|
|
32
|
+
new_file = File.join(temp_folder, add_base + SecureRandom.uuid)
|
|
32
33
|
@created_files.push(new_file)
|
|
33
|
-
|
|
34
|
+
new_file
|
|
34
35
|
end
|
|
35
36
|
|
|
36
37
|
# same as above but in global temp folder
|
|
37
38
|
def new_file_path_global(base_name)
|
|
38
|
-
username =
|
|
39
|
-
|
|
39
|
+
username = begin
|
|
40
|
+
Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
|
|
41
|
+
rescue StandardError
|
|
42
|
+
'unknown_user'
|
|
43
|
+
end
|
|
44
|
+
new_file_path_in_folder(Etc.systmpdir, base_name + '_' + username + '_')
|
|
40
45
|
end
|
|
41
46
|
|
|
42
47
|
def cleanup_expired(temp_folder)
|
|
43
48
|
# garbage collect undeleted files
|
|
44
49
|
Dir.entries(temp_folder).each do |name|
|
|
45
|
-
file_path=File.join(temp_folder,name)
|
|
46
|
-
age_sec=(Time.now - File.stat(file_path).mtime).to_i
|
|
50
|
+
file_path = File.join(temp_folder, name)
|
|
51
|
+
age_sec = (Time.now - File.stat(file_path).mtime).to_i
|
|
47
52
|
# check age of file, delete too old
|
|
48
|
-
if File.file?(file_path)
|
|
53
|
+
if File.file?(file_path) && (age_sec > FILE_LIST_AGE_MAX_SEC)
|
|
49
54
|
Log.log.debug("garbage collecting #{name}")
|
|
50
55
|
File.delete(file_path)
|
|
51
56
|
end
|
|
52
57
|
end
|
|
53
|
-
|
|
54
58
|
end
|
|
55
59
|
end
|
|
56
60
|
end
|
data/lib/aspera/timer_limiter.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
module Aspera
|
|
2
3
|
# used to throttle logs
|
|
3
4
|
class TimerLimiter
|
|
@@ -12,7 +13,7 @@ module Aspera
|
|
|
12
13
|
old_time=@last_time
|
|
13
14
|
@last_time=Time.now.to_f
|
|
14
15
|
@count+=1
|
|
15
|
-
if old_time.nil?
|
|
16
|
+
if old_time.nil? || ((@last_time-old_time)>@delay)
|
|
16
17
|
@count=0
|
|
17
18
|
return true
|
|
18
19
|
end
|
data/lib/aspera/uri_reader.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'uri'
|
|
2
3
|
require 'net/http'
|
|
3
4
|
require 'net/https'
|
|
@@ -7,16 +8,17 @@ module Aspera
|
|
|
7
8
|
# read some content from some URI, support file: , http: and https: schemes
|
|
8
9
|
def self.read(proxy_pac_uri)
|
|
9
10
|
proxy_uri=URI.parse(proxy_pac_uri)
|
|
10
|
-
|
|
11
|
+
case proxy_uri.scheme
|
|
12
|
+
when 'http'
|
|
11
13
|
return Net::HTTP.start(proxy_uri.host, proxy_uri.port){|http|http.get(proxy_uri.path)}.body
|
|
12
|
-
|
|
14
|
+
when 'https'
|
|
13
15
|
return Net::HTTPS.start(proxy_uri.host, proxy_uri.port){|http|http.get(proxy_uri.path)}.body
|
|
14
|
-
|
|
16
|
+
when 'file'
|
|
15
17
|
local_file_path=proxy_uri.path
|
|
16
|
-
raise
|
|
18
|
+
raise 'URL shall have a path, check syntax' if local_file_path.nil?
|
|
17
19
|
local_file_path=File.expand_path(local_file_path.gsub(/^\//,'')) if local_file_path.match(/^\/(~|.|..)\//)
|
|
18
20
|
return File.read(local_file_path)
|
|
19
|
-
|
|
21
|
+
when ''
|
|
20
22
|
return File.read(proxy_uri)
|
|
21
23
|
end
|
|
22
24
|
raise "no scheme: [#{proxy_uri.scheme}] for [#{proxy_pac_uri}]"
|
data/lib/aspera/web_auth.rb
CHANGED
|
@@ -1,105 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'webrick'
|
|
2
3
|
require 'webrick/https'
|
|
3
|
-
require 'thread'
|
|
4
4
|
|
|
5
5
|
module Aspera
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
6
|
+
# servlet called on callback: it records the callback request
|
|
7
|
+
class WebAuthServlet < WEBrick::HTTPServlet::AbstractServlet
|
|
8
|
+
def initialize(server,application) # additional args get here
|
|
9
|
+
Log.log.debug('WebAuthServlet initialize')
|
|
10
|
+
super(server)
|
|
11
|
+
@app=application
|
|
12
|
+
end
|
|
12
13
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
end
|
|
22
|
-
response.status=200
|
|
23
|
-
response.content_type = 'text/html'
|
|
24
|
-
response.body='<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>'
|
|
14
|
+
def service(request, response)
|
|
15
|
+
Log.log.debug("received request from browser #{request.request_method} #{request.path}")
|
|
16
|
+
raise WEBrick::HTTPStatus::MethodNotAllowed,"unexpected method: #{request.request_method}" unless request.request_method.eql?('GET')
|
|
17
|
+
raise WEBrick::HTTPStatus::NotFound,"unexpected path: #{request.path}" unless request.path.eql?(@app.expected_path)
|
|
18
|
+
# acquire lock and signal change
|
|
19
|
+
@app.mutex.synchronize do
|
|
20
|
+
@app.query=request.query
|
|
21
|
+
@app.cond.signal
|
|
25
22
|
end
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
key = OpenSSL::PKey::RSA.new(4096)
|
|
31
|
-
cert = OpenSSL::X509::Certificate.new
|
|
32
|
-
cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/C=FR/O=Test/OU=Test/CN=Test')
|
|
33
|
-
cert.not_before = Time.now
|
|
34
|
-
cert.not_after = Time.now + 365 * 24 * 60 * 60
|
|
35
|
-
cert.public_key = key.public_key
|
|
36
|
-
cert.serial = 0x0
|
|
37
|
-
cert.version = 2
|
|
38
|
-
ef = OpenSSL::X509::ExtensionFactory.new
|
|
39
|
-
ef.issuer_certificate = cert
|
|
40
|
-
ef.subject_certificate = cert
|
|
41
|
-
cert.extensions = [
|
|
42
|
-
ef.create_extension('basicConstraints','CA:TRUE', true),
|
|
43
|
-
ef.create_extension('subjectKeyIdentifier', 'hash'),
|
|
44
|
-
# ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
|
|
45
|
-
]
|
|
46
|
-
cert.add_extension(ef.create_extension('authorityKeyIdentifier','keyid:always,issuer:always'))
|
|
47
|
-
cert.sign(key, OpenSSL::Digest::SHA256.new)
|
|
48
|
-
options[:SSLPrivateKey] = key
|
|
49
|
-
options[:SSLCertificate] = cert
|
|
23
|
+
response.status=200
|
|
24
|
+
response.content_type = 'text/html'
|
|
25
|
+
response.body='<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>'
|
|
26
|
+
return nil
|
|
50
27
|
end
|
|
28
|
+
end # WebAuthServlet
|
|
29
|
+
|
|
30
|
+
# generates and adds self signed cert to provided webrick options
|
|
31
|
+
#def fill_self_signed_cert(cert,key)
|
|
32
|
+
# cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/C=FR/O=Test/OU=Test/CN=Test')
|
|
33
|
+
# cert.not_before = Time.now
|
|
34
|
+
# cert.not_after = Time.now + 365 * 24 * 60 * 60
|
|
35
|
+
# cert.public_key = key.public_key
|
|
36
|
+
# cert.serial = 0x0
|
|
37
|
+
# cert.version = 2
|
|
38
|
+
# ef = OpenSSL::X509::ExtensionFactory.new
|
|
39
|
+
# ef.issuer_certificate = cert
|
|
40
|
+
# ef.subject_certificate = cert
|
|
41
|
+
# cert.extensions = [
|
|
42
|
+
# ef.create_extension('basicConstraints','CA:TRUE', true),
|
|
43
|
+
# ef.create_extension('subjectKeyIdentifier', 'hash'),
|
|
44
|
+
# # ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
|
|
45
|
+
# ]
|
|
46
|
+
# cert.add_extension(ef.create_extension('authorityKeyIdentifier','keyid:always,issuer:always'))
|
|
47
|
+
# cert.sign(key, OpenSSL::Digest::SHA256.new)
|
|
48
|
+
#end
|
|
51
49
|
|
|
50
|
+
# start a local web server, then start a browser that will callback the local server upon authentication
|
|
51
|
+
class WebAuth
|
|
52
|
+
attr_reader :expected_path,:mutex,:cond
|
|
53
|
+
attr_writer :query
|
|
54
|
+
# @param endpoint_url [String] e.g. 'https://127.0.0.1:12345'
|
|
52
55
|
def initialize(endpoint_url)
|
|
53
56
|
uri=URI.parse(endpoint_url)
|
|
57
|
+
# parameters for servlet
|
|
58
|
+
@query=nil
|
|
59
|
+
@mutex=Mutex.new
|
|
60
|
+
@cond=ConditionVariable.new
|
|
61
|
+
@expected_path=uri.path.empty? ? '/' : uri.path
|
|
62
|
+
# see https://www.rubydoc.info/stdlib/webrick/WEBrick/Config
|
|
54
63
|
webrick_options = {
|
|
55
|
-
:
|
|
56
|
-
:
|
|
57
|
-
:
|
|
64
|
+
BindAddress: uri.host,
|
|
65
|
+
Port: uri.port,
|
|
66
|
+
Logger: Log.log,
|
|
67
|
+
AccessLog: [[self, WEBrick::AccessLog::COMMON_LOG_FORMAT]] # see "<<" below
|
|
58
68
|
}
|
|
59
|
-
uri_path=uri.path.empty? ? '/' : uri.path
|
|
60
69
|
case uri.scheme
|
|
61
70
|
when 'http'
|
|
62
71
|
Log.log.debug('HTTP mode')
|
|
63
72
|
when 'https'
|
|
64
73
|
webrick_options[:SSLEnable]=true
|
|
65
|
-
webrick_options[:SSLVerifyClient]=OpenSSL::SSL::VERIFY_NONE
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
# good cert
|
|
76
|
-
webrick_options[:SSLPrivateKey] =OpenSSL::PKey::RSA.new(File.read('/Users/laurent/workspace/Tools/certificate/myserver.key'))
|
|
77
|
-
webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read('/Users/laurent/workspace/Tools/certificate/myserver.crt'))
|
|
78
|
-
end
|
|
74
|
+
webrick_options[:SSLVerifyClient] = OpenSSL::SSL::VERIFY_NONE
|
|
75
|
+
# a- automatic certificate generation
|
|
76
|
+
webrick_options[:SSLCertName] = [['CN',WEBrick::Utils.getservername]]
|
|
77
|
+
# b- generate self signed cert
|
|
78
|
+
#webrick_options[:SSLPrivateKey] = OpenSSL::PKey::RSA.new(4096)
|
|
79
|
+
#webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new
|
|
80
|
+
#self.class.fill_self_signed_cert(webrick_options[:SSLCertificate],webrick_options[:SSLPrivateKey])
|
|
81
|
+
## c- good cert
|
|
82
|
+
#webrick_options[:SSLPrivateKey] = OpenSSL::PKey::RSA.new(File.read('.../myserver.key'))
|
|
83
|
+
#webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read('.../myserver.crt'))
|
|
79
84
|
end
|
|
80
|
-
# parameters for servlet
|
|
81
|
-
@shared_info={
|
|
82
|
-
expected_path: uri_path,
|
|
83
|
-
mutex: Mutex.new,
|
|
84
|
-
cond: ConditionVariable.new
|
|
85
|
-
}
|
|
86
85
|
@server = WEBrick::HTTPServer.new(webrick_options)
|
|
87
|
-
@server.mount(
|
|
86
|
+
@server.mount(@expected_path, WebAuthServlet, self) # additional args provided to constructor
|
|
88
87
|
Thread.new { @server.start }
|
|
89
88
|
end
|
|
90
89
|
|
|
90
|
+
# log web server access
|
|
91
|
+
def <<(access_log)
|
|
92
|
+
Log.log.debug{"webrick log #{access_log.chomp}"}
|
|
93
|
+
end
|
|
94
|
+
|
|
91
95
|
# wait for request on web server
|
|
92
96
|
# @return Hash the query
|
|
93
|
-
def
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
@
|
|
98
|
-
|
|
99
|
-
end
|
|
97
|
+
def received_request
|
|
98
|
+
# shall be called only once
|
|
99
|
+
raise 'error, received_request called twice ?' if @server.nil?
|
|
100
|
+
# wait for signal from thread
|
|
101
|
+
@mutex.synchronize{@cond.wait(@mutex)}
|
|
102
|
+
# tell server thread to stop
|
|
100
103
|
@server.shutdown
|
|
101
104
|
@server=nil
|
|
102
|
-
return @
|
|
105
|
+
return @query
|
|
103
106
|
end
|
|
104
107
|
end
|
|
105
108
|
end
|