aspera-cli 4.5.0 → 4.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +1 -0
- data/README.md +1894 -1574
- data/bin/ascli +21 -1
- data/bin/asession +38 -34
- data/docs/test_env.conf +14 -3
- data/examples/aoc.rb +17 -15
- data/examples/dascli +26 -0
- data/examples/faspex4.rb +42 -35
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +38 -37
- data/lib/aspera/aoc.rb +245 -205
- data/lib/aspera/ascmd.rb +111 -90
- data/lib/aspera/ats_api.rb +16 -14
- data/lib/aspera/cli/basic_auth_plugin.rb +19 -18
- data/lib/aspera/cli/extended_value.rb +50 -39
- data/lib/aspera/cli/formater.rb +161 -135
- data/lib/aspera/cli/info.rb +18 -0
- data/lib/aspera/cli/listener/line_dump.rb +4 -2
- data/lib/aspera/cli/listener/logger.rb +3 -1
- data/lib/aspera/cli/listener/progress.rb +20 -21
- data/lib/aspera/cli/listener/progress_multi.rb +29 -31
- data/lib/aspera/cli/main.rb +194 -183
- data/lib/aspera/cli/manager.rb +213 -206
- data/lib/aspera/cli/plugin.rb +71 -49
- data/lib/aspera/cli/plugins/alee.rb +8 -7
- data/lib/aspera/cli/plugins/aoc.rb +675 -558
- data/lib/aspera/cli/plugins/ats.rb +116 -109
- data/lib/aspera/cli/plugins/bss.rb +35 -34
- data/lib/aspera/cli/plugins/config.rb +722 -542
- data/lib/aspera/cli/plugins/console.rb +28 -22
- data/lib/aspera/cli/plugins/cos.rb +28 -37
- data/lib/aspera/cli/plugins/faspex.rb +281 -227
- data/lib/aspera/cli/plugins/faspex5.rb +129 -84
- data/lib/aspera/cli/plugins/node.rb +426 -232
- data/lib/aspera/cli/plugins/orchestrator.rb +106 -98
- data/lib/aspera/cli/plugins/preview.rb +196 -191
- data/lib/aspera/cli/plugins/server.rb +131 -126
- data/lib/aspera/cli/plugins/shares.rb +49 -36
- data/lib/aspera/cli/plugins/sync.rb +27 -28
- data/lib/aspera/cli/transfer_agent.rb +84 -79
- data/lib/aspera/cli/version.rb +3 -1
- data/lib/aspera/colors.rb +37 -28
- data/lib/aspera/command_line_builder.rb +84 -63
- data/lib/aspera/cos_node.rb +68 -34
- data/lib/aspera/data_repository.rb +4 -2
- data/lib/aspera/environment.rb +61 -46
- data/lib/aspera/fasp/agent_base.rb +36 -31
- data/lib/aspera/fasp/agent_connect.rb +44 -37
- data/lib/aspera/fasp/agent_direct.rb +101 -104
- data/lib/aspera/fasp/agent_httpgw.rb +91 -90
- data/lib/aspera/fasp/agent_node.rb +36 -33
- data/lib/aspera/fasp/agent_trsdk.rb +28 -31
- data/lib/aspera/fasp/error.rb +3 -1
- data/lib/aspera/fasp/error_info.rb +81 -54
- data/lib/aspera/fasp/installation.rb +171 -151
- data/lib/aspera/fasp/listener.rb +2 -0
- data/lib/aspera/fasp/parameters.rb +105 -111
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +20 -20
- data/lib/aspera/fasp/transfer_spec.rb +27 -0
- data/lib/aspera/fasp/uri.rb +31 -29
- data/lib/aspera/faspex_gw.rb +95 -118
- data/lib/aspera/hash_ext.rb +12 -13
- data/lib/aspera/id_generator.rb +11 -9
- data/lib/aspera/keychain/encrypted_hash.rb +73 -57
- data/lib/aspera/keychain/macos_security.rb +27 -29
- data/lib/aspera/log.rb +40 -39
- data/lib/aspera/nagios.rb +24 -22
- data/lib/aspera/node.rb +38 -30
- data/lib/aspera/oauth.rb +217 -248
- data/lib/aspera/open_application.rb +9 -7
- data/lib/aspera/persistency_action_once.rb +15 -14
- data/lib/aspera/persistency_folder.rb +15 -18
- data/lib/aspera/preview/file_types.rb +266 -270
- data/lib/aspera/preview/generator.rb +94 -92
- data/lib/aspera/preview/image_error.png +0 -0
- data/lib/aspera/preview/options.rb +20 -17
- data/lib/aspera/preview/utils.rb +99 -102
- data/lib/aspera/preview/video_error.png +0 -0
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +114 -21
- data/lib/aspera/rest.rb +144 -142
- data/lib/aspera/rest_call_error.rb +3 -2
- data/lib/aspera/rest_error_analyzer.rb +31 -31
- data/lib/aspera/rest_errors_aspera.rb +18 -16
- data/lib/aspera/secret_hider.rb +68 -0
- data/lib/aspera/ssh.rb +20 -16
- data/lib/aspera/sync.rb +57 -54
- data/lib/aspera/temp_file_manager.rb +20 -14
- data/lib/aspera/timer_limiter.rb +10 -8
- data/lib/aspera/uri_reader.rb +14 -15
- data/lib/aspera/web_auth.rb +85 -80
- data.tar.gz.sig +0 -0
- metadata +169 -40
- metadata.gz.sig +2 -0
- data/bin/dascli +0 -13
- data/docs/Makefile +0 -63
- data/docs/README.erb.md +0 -4221
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'logger'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
# remove secret from logs and output
|
7
|
+
class SecretHider
|
8
|
+
# display string for hidden secrets
|
9
|
+
HIDDEN_PASSWORD = '🔑'
|
10
|
+
# keys in hash that contain secrets
|
11
|
+
SECRET_KEYWORDS = %w[password secret private_key passphrase].freeze
|
12
|
+
# regex that define namec captures :begin and :end
|
13
|
+
REGEX_LOG_REPLACES=[
|
14
|
+
# replace values in logs with rendered JSON
|
15
|
+
/(?<begin>["':][^"]*(#{SECRET_KEYWORDS.join('|')})[^"]*["']?[=>: ]+")[^"]+(?<end>")/,
|
16
|
+
# option "secret"
|
17
|
+
/(?<begin>"[^"]*(secret)[^"]*"=>{)[^}]+(?<end>})/,
|
18
|
+
# option "secrets"
|
19
|
+
/(?<begin>(secrets)={)[^}]+(?<end>})/,
|
20
|
+
# private key values
|
21
|
+
/(?<begin>--+BEGIN .+ KEY--+)[[:ascii:]]+?(?<end>--+?END .+ KEY--+)/
|
22
|
+
].freeze
|
23
|
+
private_constant :HIDDEN_PASSWORD,:SECRET_KEYWORDS
|
24
|
+
@log_secrets = false
|
25
|
+
class << self
|
26
|
+
attr_accessor :log_secrets
|
27
|
+
def log_formatter(original_formatter)
|
28
|
+
original_formatter ||= Logger::Formatter.new
|
29
|
+
# note that @log_secrets may be set AFTER this init is done, so it's done at runtime
|
30
|
+
return lambda do |severity, datetime, progname, msg|
|
31
|
+
if msg.is_a?(String) && !@log_secrets
|
32
|
+
REGEX_LOG_REPLACES.each do |regx|
|
33
|
+
msg = msg.gsub(regx){"#{Regexp.last_match(:begin)}#{HIDDEN_PASSWORD}#{Regexp.last_match(:end)}"}
|
34
|
+
end
|
35
|
+
end
|
36
|
+
original_formatter.call(severity, datetime, progname, msg)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def secret?(keyword,value)
|
41
|
+
keyword=keyword.to_s if keyword.is_a?(Symbol)
|
42
|
+
# only Strings can be secrets, not booleans, or hash, arrays
|
43
|
+
keyword.is_a?(String) && SECRET_KEYWORDS.any?{|kw|keyword.include?(kw)} && value.is_a?(String)
|
44
|
+
end
|
45
|
+
|
46
|
+
def deep_remove_secret(obj,is_name_value: false)
|
47
|
+
case obj
|
48
|
+
when Array
|
49
|
+
if is_name_value
|
50
|
+
obj.each do |i|
|
51
|
+
i['value']=HIDDEN_PASSWORD if secret?(i['parameter'],i['value'])
|
52
|
+
end
|
53
|
+
else
|
54
|
+
obj.each{|i|deep_remove_secret(i)}
|
55
|
+
end
|
56
|
+
when Hash
|
57
|
+
obj.each do |k,v|
|
58
|
+
if secret?(k,v)
|
59
|
+
obj[k] = HIDDEN_PASSWORD
|
60
|
+
elsif obj[k].is_a?(Hash)
|
61
|
+
deep_remove_secret(obj[k])
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
data/lib/aspera/ssh.rb
CHANGED
@@ -1,9 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'net/ssh'
|
2
4
|
|
3
5
|
# Hack: deactivate ed25519 and ecdsa private keys from ssh identities, as it usually hurts
|
4
6
|
begin
|
5
|
-
module Net;
|
6
|
-
rescue
|
7
|
+
module Net;module SSH;module Authentication;class Session;private; def default_keys; %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa];end;end;end;end;end # rubocop:disable Layout/AccessModifierIndentation, Layout/EmptyLinesAroundAccessModifier, Layout/LineLength
|
8
|
+
rescue StandardError
|
9
|
+
# ignore errors
|
7
10
|
end
|
8
11
|
|
9
12
|
module Aspera
|
@@ -14,40 +17,41 @@ module Aspera
|
|
14
17
|
# see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
|
15
18
|
def initialize(host,username,ssh_options)
|
16
19
|
Log.log.debug("ssh:#{username}@#{host}")
|
17
|
-
@host=host
|
18
|
-
@username=username
|
19
|
-
@ssh_options=ssh_options
|
20
|
-
@ssh_options[:logger]=Log.log
|
20
|
+
@host = host
|
21
|
+
@username = username
|
22
|
+
@ssh_options = ssh_options
|
23
|
+
@ssh_options[:logger] = Log.log
|
21
24
|
end
|
22
25
|
|
23
26
|
def execute(cmd,input=nil)
|
24
27
|
if cmd.is_a?(Array)
|
25
28
|
# concatenate arguments, enclose in double quotes
|
26
|
-
cmd=cmd.map{|v
|
29
|
+
cmd = cmd.map{|v|%Q("#{v}")}.join(' ')
|
27
30
|
end
|
28
31
|
Log.log.debug("cmd=#{cmd}")
|
29
|
-
response =
|
32
|
+
response = []
|
30
33
|
Net::SSH.start(@host, @username, @ssh_options) do |session|
|
31
|
-
ssh_channel=session.open_channel do |channel|
|
34
|
+
ssh_channel = session.open_channel do |channel|
|
32
35
|
# prepare stdout processing
|
33
|
-
channel.on_data{|
|
36
|
+
channel.on_data{|_chan,data|response.push(data)}
|
34
37
|
# prepare stderr processing, stderr if type = 1
|
35
|
-
channel.on_extended_data do |
|
36
|
-
errormsg="#{cmd}: [#{data.chomp}]"
|
38
|
+
channel.on_extended_data do |_chan, _type, data|
|
39
|
+
errormsg = "#{cmd}: [#{data.chomp}]"
|
37
40
|
# Happens when windows user hasn't logged in and created home account.
|
38
|
-
if data.include?(
|
39
|
-
errormsg
|
41
|
+
if data.include?('Could not chdir to home directory')
|
42
|
+
errormsg += "\nHint: home not created in Windows?"
|
40
43
|
end
|
41
44
|
raise errormsg
|
42
45
|
end
|
43
|
-
channel
|
46
|
+
# send commannd to SSH channel (execute)
|
47
|
+
channel.send('cexe'.reverse,cmd){|_ch,_success|channel.send_data(input) unless input.nil?}
|
44
48
|
end
|
45
49
|
# wait for channel to finish (command exit)
|
46
50
|
ssh_channel.wait
|
47
51
|
# main ssh session loop
|
48
52
|
session.loop
|
49
53
|
end # session
|
50
|
-
return response
|
54
|
+
return response.join
|
51
55
|
end
|
52
56
|
end
|
53
57
|
end
|
data/lib/aspera/sync.rb
CHANGED
@@ -1,48 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'aspera/command_line_builder'
|
2
4
|
|
3
5
|
module Aspera
|
4
6
|
# builds command line arg for async
|
5
7
|
class Sync
|
6
|
-
INSTANCE_PARAMS=
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
SESSION_PARAMS=
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
8
|
+
INSTANCE_PARAMS =
|
9
|
+
{
|
10
|
+
'alt_logdir' => { cltype: :opt_with_arg, accepted_types: :string},
|
11
|
+
'watchd' => { cltype: :opt_with_arg, accepted_types: :string},
|
12
|
+
'apply_local_docroot' => { cltype: :opt_without_arg},
|
13
|
+
'quiet' => { cltype: :opt_without_arg}
|
14
|
+
}.freeze
|
15
|
+
SESSION_PARAMS =
|
16
|
+
{
|
17
|
+
'name' => { cltype: :opt_with_arg, accepted_types: :string},
|
18
|
+
'local_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
19
|
+
'remote_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
20
|
+
'local_db_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
21
|
+
'remote_db_dir' => { cltype: :opt_with_arg, accepted_types: :string},
|
22
|
+
'host' => { cltype: :opt_with_arg, accepted_types: :string},
|
23
|
+
'user' => { cltype: :opt_with_arg, accepted_types: :string},
|
24
|
+
'private_key_path' => { cltype: :opt_with_arg, accepted_types: :string},
|
25
|
+
'direction' => { cltype: :opt_with_arg, accepted_types: :string},
|
26
|
+
'checksum' => { cltype: :opt_with_arg, accepted_types: :string},
|
27
|
+
'tcp_port' => { cltype: :opt_with_arg, accepted_types: :int},
|
28
|
+
'rate_policy' => { cltype: :opt_with_arg, accepted_types: :string},
|
29
|
+
'target_rate' => { cltype: :opt_with_arg, accepted_types: :string},
|
30
|
+
'cooloff' => { cltype: :opt_with_arg, accepted_types: :int},
|
31
|
+
'pending_max' => { cltype: :opt_with_arg, accepted_types: :int},
|
32
|
+
'scan_intensity' => { cltype: :opt_with_arg, accepted_types: :string},
|
33
|
+
'cipher' => { cltype: :opt_with_arg, accepted_types: :string},
|
34
|
+
'transfer_threads' => { cltype: :opt_with_arg, accepted_types: :int},
|
35
|
+
'preserve_time' => { cltype: :opt_without_arg},
|
36
|
+
'preserve_access_time' => { cltype: :opt_without_arg},
|
37
|
+
'preserve_modification_time' => { cltype: :opt_without_arg},
|
38
|
+
'preserve_uid' => { cltype: :opt_without_arg},
|
39
|
+
'preserve_gid' => { cltype: :opt_without_arg},
|
40
|
+
'create_dir' => { cltype: :opt_without_arg},
|
41
|
+
'reset' => { cltype: :opt_without_arg},
|
42
|
+
# note: only one env var, but multiple sessions... may be a problem
|
43
|
+
'remote_password' => { cltype: :envvar, clvarname: 'ASPERA_SCP_PASS'},
|
44
|
+
'cookie' => { cltype: :envvar, clvarname: 'ASPERA_SCP_COOKIE'},
|
45
|
+
'token' => { cltype: :envvar, clvarname: 'ASPERA_SCP_TOKEN'},
|
46
|
+
'license' => { cltype: :envvar, clvarname: 'ASPERA_SCP_LICENSE'}
|
47
|
+
}.freeze
|
46
48
|
|
47
49
|
Aspera::CommandLineBuilder.normalize_description(INSTANCE_PARAMS)
|
48
50
|
Aspera::CommandLineBuilder.normalize_description(SESSION_PARAMS)
|
@@ -50,33 +52,34 @@ module Aspera
|
|
50
52
|
private_constant :INSTANCE_PARAMS,:SESSION_PARAMS
|
51
53
|
|
52
54
|
def initialize(sync_params)
|
53
|
-
@sync_params=sync_params
|
55
|
+
@sync_params = sync_params
|
54
56
|
end
|
55
57
|
|
56
|
-
MANDATORY_KEYS=[
|
58
|
+
MANDATORY_KEYS = %w[instance sessions].freeze
|
57
59
|
|
58
60
|
def compute_args
|
59
|
-
raise StandardError,
|
60
|
-
raise StandardError,"parameter hash must have at least 'sessions', and optionally 'instance' keys." unless
|
61
|
-
|
62
|
-
raise StandardError,
|
61
|
+
raise StandardError,'parameter must be Hash' unless @sync_params.is_a?(Hash)
|
62
|
+
raise StandardError,"parameter hash must have at least 'sessions', and optionally 'instance' keys." unless
|
63
|
+
@sync_params.keys.push('instance').uniq.sort.eql?(MANDATORY_KEYS)
|
64
|
+
raise StandardError,'sessions key must be Array' unless @sync_params['sessions'].is_a?(Array)
|
65
|
+
raise StandardError,'sessions key must has at least one element (hash)' unless @sync_params['sessions'].first.is_a?(Hash)
|
63
66
|
|
64
|
-
env_args={
|
65
|
-
:
|
66
|
-
:
|
67
|
+
env_args = {
|
68
|
+
args: [],
|
69
|
+
env: {}
|
67
70
|
}
|
68
71
|
|
69
72
|
if @sync_params.has_key?('instance')
|
70
|
-
raise StandardError,
|
71
|
-
instance_builder=CommandLineBuilder.new(@sync_params['instance'],INSTANCE_PARAMS)
|
73
|
+
raise StandardError,'instance key must be hash' unless @sync_params['instance'].is_a?(Hash)
|
74
|
+
instance_builder = CommandLineBuilder.new(@sync_params['instance'],INSTANCE_PARAMS)
|
72
75
|
instance_builder.process_params
|
73
76
|
instance_builder.add_env_args(env_args[:env],env_args[:args])
|
74
77
|
end
|
75
78
|
|
76
79
|
@sync_params['sessions'].each do |session_params|
|
77
|
-
raise StandardError,
|
78
|
-
raise StandardError,
|
79
|
-
session_builder=CommandLineBuilder.new(session_params,SESSION_PARAMS)
|
80
|
+
raise StandardError,'sessions must contain hashes' unless session_params.is_a?(Hash)
|
81
|
+
raise StandardError,'session must contain at leat name' unless session_params.has_key?('name')
|
82
|
+
session_builder = CommandLineBuilder.new(session_params,SESSION_PARAMS)
|
80
83
|
session_builder.process_params
|
81
84
|
session_builder.add_env_args(env_args[:env],env_args[:args])
|
82
85
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'singleton'
|
2
4
|
require 'fileutils'
|
3
5
|
require 'etc'
|
@@ -6,14 +8,14 @@ module Aspera
|
|
6
8
|
# create a temp file name for a given folder
|
7
9
|
# files can be deleted on process exit by calling cleanup
|
8
10
|
class TempFileManager
|
9
|
-
SEC_IN_DAY=
|
11
|
+
SEC_IN_DAY = 86_400
|
10
12
|
# assume no transfer last longer than this
|
11
13
|
# (garbage collect file list which were not deleted after transfer)
|
12
|
-
FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
|
13
|
-
private_constant :SEC_IN_DAY
|
14
|
+
FILE_LIST_AGE_MAX_SEC = 5 * SEC_IN_DAY
|
15
|
+
private_constant :SEC_IN_DAY, :FILE_LIST_AGE_MAX_SEC
|
14
16
|
include Singleton
|
15
17
|
def initialize
|
16
|
-
@created_files=[]
|
18
|
+
@created_files = []
|
17
19
|
end
|
18
20
|
|
19
21
|
# call this on process exit
|
@@ -21,36 +23,40 @@ module Aspera
|
|
21
23
|
@created_files.each do |filepath|
|
22
24
|
File.delete(filepath) if File.file?(filepath)
|
23
25
|
end
|
24
|
-
@created_files=[]
|
26
|
+
@created_files = []
|
25
27
|
end
|
26
28
|
|
27
29
|
# ensure that provided folder exists, or create it, generate a unique filename
|
28
30
|
# @return path to that unique file
|
29
|
-
def new_file_path_in_folder(temp_folder,add_base='')
|
31
|
+
def new_file_path_in_folder(temp_folder, add_base = '')
|
30
32
|
FileUtils.mkdir_p(temp_folder) unless Dir.exist?(temp_folder)
|
31
|
-
new_file=File.join(temp_folder,add_base+SecureRandom.uuid)
|
33
|
+
new_file = File.join(temp_folder, add_base + SecureRandom.uuid)
|
32
34
|
@created_files.push(new_file)
|
33
|
-
|
35
|
+
new_file
|
34
36
|
end
|
35
37
|
|
36
38
|
# same as above but in global temp folder
|
37
39
|
def new_file_path_global(base_name)
|
38
|
-
username =
|
39
|
-
|
40
|
+
username =
|
41
|
+
begin
|
42
|
+
Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
|
43
|
+
rescue StandardError
|
44
|
+
'unknown_user'
|
45
|
+
end
|
46
|
+
new_file_path_in_folder(Etc.systmpdir, base_name + '_' + username + '_')
|
40
47
|
end
|
41
48
|
|
42
49
|
def cleanup_expired(temp_folder)
|
43
50
|
# garbage collect undeleted files
|
44
51
|
Dir.entries(temp_folder).each do |name|
|
45
|
-
file_path=File.join(temp_folder,name)
|
46
|
-
age_sec=(Time.now - File.stat(file_path).mtime).to_i
|
52
|
+
file_path = File.join(temp_folder, name)
|
53
|
+
age_sec = (Time.now - File.stat(file_path).mtime).to_i
|
47
54
|
# check age of file, delete too old
|
48
|
-
if File.file?(file_path)
|
55
|
+
if File.file?(file_path) && (age_sec > FILE_LIST_AGE_MAX_SEC)
|
49
56
|
Log.log.debug("garbage collecting #{name}")
|
50
57
|
File.delete(file_path)
|
51
58
|
end
|
52
59
|
end
|
53
|
-
|
54
60
|
end
|
55
61
|
end
|
56
62
|
end
|
data/lib/aspera/timer_limiter.rb
CHANGED
@@ -1,19 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Aspera
|
2
4
|
# used to throttle logs
|
3
5
|
class TimerLimiter
|
4
6
|
# @param delay in seconds (float)
|
5
7
|
def initialize(delay)
|
6
|
-
@delay=delay
|
7
|
-
@last_time=nil
|
8
|
-
@count=0
|
8
|
+
@delay = delay
|
9
|
+
@last_time = nil
|
10
|
+
@count = 0
|
9
11
|
end
|
10
12
|
|
11
13
|
def trigger?
|
12
|
-
old_time
|
13
|
-
@last_time=Time.now.to_f
|
14
|
-
@count+=1
|
15
|
-
if old_time.nil?
|
16
|
-
@count=0
|
14
|
+
old_time = @last_time
|
15
|
+
@last_time = Time.now.to_f
|
16
|
+
@count += 1
|
17
|
+
if old_time.nil? || ((@last_time - old_time) > @delay)
|
18
|
+
@count = 0
|
17
19
|
return true
|
18
20
|
end
|
19
21
|
return false
|
data/lib/aspera/uri_reader.rb
CHANGED
@@ -1,25 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'uri'
|
2
|
-
require '
|
3
|
-
require 'net/https'
|
4
|
+
require 'aspera/rest'
|
4
5
|
|
5
6
|
module Aspera
|
6
7
|
module UriReader
|
7
8
|
# read some content from some URI, support file: , http: and https: schemes
|
8
|
-
def self.read(
|
9
|
-
proxy_uri=URI.parse(
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
local_file_path
|
16
|
-
|
17
|
-
local_file_path=File.expand_path(local_file_path.gsub(/^\//,'')) if local_file_path.match(/^\/(~|.|..)\//)
|
9
|
+
def self.read(uri_to_read)
|
10
|
+
proxy_uri = URI.parse(uri_to_read)
|
11
|
+
case proxy_uri.scheme
|
12
|
+
when 'http','https'
|
13
|
+
return Rest.new(base_url: uri_to_read,redirect_max: 5).call(operation: 'GET', subpath: '', headers: {'Accept' => 'text/plain'})[:data]
|
14
|
+
when 'file',NilClass
|
15
|
+
local_file_path = proxy_uri.path
|
16
|
+
raise 'URL shall have a path, check syntax' if local_file_path.nil?
|
17
|
+
local_file_path = File.expand_path(local_file_path.gsub(/^\//,'')) if /^\/(~|.|..)\//.match?(local_file_path)
|
18
18
|
return File.read(local_file_path)
|
19
|
-
|
20
|
-
|
19
|
+
else
|
20
|
+
raise "unknown scheme: [#{proxy_uri.scheme}] for [#{uri_to_read}]"
|
21
21
|
end
|
22
|
-
raise "no scheme: [#{proxy_uri.scheme}] for [#{proxy_pac_uri}]"
|
23
22
|
end
|
24
23
|
end
|
25
24
|
end
|
data/lib/aspera/web_auth.rb
CHANGED
@@ -1,105 +1,110 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'webrick'
|
2
4
|
require 'webrick/https'
|
3
|
-
require '
|
5
|
+
require 'stringio'
|
4
6
|
|
5
7
|
module Aspera
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
8
|
+
# servlet called on callback: it records the callback request
|
9
|
+
class WebAuthServlet < WEBrick::HTTPServlet::AbstractServlet
|
10
|
+
def initialize(server,application) # additional args get here
|
11
|
+
Log.log.debug('WebAuthServlet initialize')
|
12
|
+
super(server)
|
13
|
+
@app = application
|
14
|
+
end
|
12
15
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
end
|
22
|
-
response.status=200
|
23
|
-
response.content_type = 'text/html'
|
24
|
-
response.body='<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>'
|
16
|
+
def service(request, response)
|
17
|
+
Log.log.debug("received request from browser #{request.request_method} #{request.path}")
|
18
|
+
raise WEBrick::HTTPStatus::MethodNotAllowed,"unexpected method: #{request.request_method}" unless request.request_method.eql?('GET')
|
19
|
+
raise WEBrick::HTTPStatus::NotFound,"unexpected path: #{request.path}" unless request.path.eql?(@app.expected_path)
|
20
|
+
# acquire lock and signal change
|
21
|
+
@app.mutex.synchronize do
|
22
|
+
@app.query = request.query
|
23
|
+
@app.cond.signal
|
25
24
|
end
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
key = OpenSSL::PKey::RSA.new(4096)
|
31
|
-
cert = OpenSSL::X509::Certificate.new
|
32
|
-
cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/C=FR/O=Test/OU=Test/CN=Test')
|
33
|
-
cert.not_before = Time.now
|
34
|
-
cert.not_after = Time.now + 365 * 24 * 60 * 60
|
35
|
-
cert.public_key = key.public_key
|
36
|
-
cert.serial = 0x0
|
37
|
-
cert.version = 2
|
38
|
-
ef = OpenSSL::X509::ExtensionFactory.new
|
39
|
-
ef.issuer_certificate = cert
|
40
|
-
ef.subject_certificate = cert
|
41
|
-
cert.extensions = [
|
42
|
-
ef.create_extension('basicConstraints','CA:TRUE', true),
|
43
|
-
ef.create_extension('subjectKeyIdentifier', 'hash'),
|
44
|
-
# ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
|
45
|
-
]
|
46
|
-
cert.add_extension(ef.create_extension('authorityKeyIdentifier','keyid:always,issuer:always'))
|
47
|
-
cert.sign(key, OpenSSL::Digest::SHA256.new)
|
48
|
-
options[:SSLPrivateKey] = key
|
49
|
-
options[:SSLCertificate] = cert
|
25
|
+
response.status = 200
|
26
|
+
response.content_type = 'text/html'
|
27
|
+
response.body = '<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>'
|
28
|
+
return nil
|
50
29
|
end
|
30
|
+
end # WebAuthServlet
|
31
|
+
|
32
|
+
# generates and adds self signed cert to provided webrick options
|
33
|
+
#def fill_self_signed_cert(cert,key)
|
34
|
+
# cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/C=FR/O=Test/OU=Test/CN=Test')
|
35
|
+
# cert.not_before = Time.now
|
36
|
+
# cert.not_after = Time.now + 365 * 24 * 60 * 60
|
37
|
+
# cert.public_key = key.public_key
|
38
|
+
# cert.serial = 0x0
|
39
|
+
# cert.version = 2
|
40
|
+
# ef = OpenSSL::X509::ExtensionFactory.new
|
41
|
+
# ef.issuer_certificate = cert
|
42
|
+
# ef.subject_certificate = cert
|
43
|
+
# cert.extensions = [
|
44
|
+
# ef.create_extension('basicConstraints','CA:TRUE', true),
|
45
|
+
# ef.create_extension('subjectKeyIdentifier', 'hash'),
|
46
|
+
# # ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
|
47
|
+
# ]
|
48
|
+
# cert.add_extension(ef.create_extension('authorityKeyIdentifier','keyid:always,issuer:always'))
|
49
|
+
# cert.sign(key, OpenSSL::Digest::SHA256.new)
|
50
|
+
#end
|
51
51
|
|
52
|
+
# start a local web server, then start a browser that will callback the local server upon authentication
|
53
|
+
class WebAuth
|
54
|
+
attr_reader :expected_path,:mutex,:cond
|
55
|
+
attr_writer :query
|
56
|
+
# @param endpoint_url [String] e.g. 'https://127.0.0.1:12345'
|
52
57
|
def initialize(endpoint_url)
|
53
|
-
uri=URI.parse(endpoint_url)
|
58
|
+
uri = URI.parse(endpoint_url)
|
59
|
+
# parameters for servlet
|
60
|
+
@query = nil
|
61
|
+
@mutex = Mutex.new
|
62
|
+
@cond = ConditionVariable.new
|
63
|
+
@expected_path = uri.path.empty? ? '/' : uri.path
|
64
|
+
# see https://www.rubydoc.info/stdlib/webrick/WEBrick/Config
|
54
65
|
webrick_options = {
|
55
|
-
:
|
56
|
-
:
|
57
|
-
:
|
66
|
+
BindAddress: uri.host,
|
67
|
+
Port: uri.port,
|
68
|
+
Logger: Log.log,
|
69
|
+
AccessLog: [[self, WEBrick::AccessLog::COMMON_LOG_FORMAT]] # replace default access log to call local method "<<" below
|
58
70
|
}
|
59
|
-
uri_path=uri.path.empty? ? '/' : uri.path
|
60
71
|
case uri.scheme
|
61
72
|
when 'http'
|
62
73
|
Log.log.debug('HTTP mode')
|
63
74
|
when 'https'
|
64
|
-
webrick_options[:SSLEnable]=true
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
when 2
|
75
|
-
# good cert
|
76
|
-
webrick_options[:SSLPrivateKey] =OpenSSL::PKey::RSA.new(File.read('/Users/laurent/workspace/Tools/certificate/myserver.key'))
|
77
|
-
webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read('/Users/laurent/workspace/Tools/certificate/myserver.crt'))
|
78
|
-
end
|
75
|
+
webrick_options[:SSLEnable] = true
|
76
|
+
# a- automatic certificate generation
|
77
|
+
webrick_options[:SSLCertName] = [['CN',WEBrick::Utils.getservername]]
|
78
|
+
# b- generate self signed cert
|
79
|
+
#webrick_options[:SSLPrivateKey] = OpenSSL::PKey::RSA.new(4096)
|
80
|
+
#webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new
|
81
|
+
#self.class.fill_self_signed_cert(webrick_options[:SSLCertificate],webrick_options[:SSLPrivateKey])
|
82
|
+
## c- good cert
|
83
|
+
#webrick_options[:SSLPrivateKey] = OpenSSL::PKey::RSA.new(File.read('.../myserver.key'))
|
84
|
+
#webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read('.../myserver.crt'))
|
79
85
|
end
|
80
|
-
#
|
81
|
-
@
|
82
|
-
|
83
|
-
mutex: Mutex.new,
|
84
|
-
cond: ConditionVariable.new
|
85
|
-
}
|
86
|
-
@server = WEBrick::HTTPServer.new(webrick_options)
|
87
|
-
@server.mount(uri_path, FxGwServlet, @shared_info) # additional args provided to constructor
|
86
|
+
# self signed certificate generates characters on STDERR, see create_self_signed_cert in webrick/ssl.rb
|
87
|
+
Log.capture_stderr { @server = WEBrick::HTTPServer.new(webrick_options) }
|
88
|
+
@server.mount(@expected_path, WebAuthServlet, self) # additional args provided to constructor
|
88
89
|
Thread.new { @server.start }
|
89
90
|
end
|
90
91
|
|
92
|
+
# log web server access ( option AccessLog )
|
93
|
+
def <<(access_log)
|
94
|
+
Log.log.debug{"webrick log #{access_log.chomp}"}
|
95
|
+
end
|
96
|
+
|
91
97
|
# wait for request on web server
|
92
98
|
# @return Hash the query
|
93
|
-
def
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
@
|
98
|
-
|
99
|
-
end
|
99
|
+
def received_request
|
100
|
+
# shall be called only once
|
101
|
+
raise 'error, received_request called twice ?' if @server.nil?
|
102
|
+
# wait for signal from thread
|
103
|
+
@mutex.synchronize{@cond.wait(@mutex)}
|
104
|
+
# tell server thread to stop
|
100
105
|
@server.shutdown
|
101
|
-
@server=nil
|
102
|
-
return @
|
106
|
+
@server = nil
|
107
|
+
return @query
|
103
108
|
end
|
104
109
|
end
|
105
110
|
end
|
data.tar.gz.sig
ADDED
Binary file
|