aspera-cli 4.6.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
data/lib/aspera/log.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/colors'
|
2
3
|
require 'logger'
|
3
4
|
require 'pp'
|
@@ -7,6 +8,9 @@ require 'singleton'
|
|
7
8
|
module Aspera
|
8
9
|
# Singleton object for logging
|
9
10
|
class Log
|
11
|
+
# display string for hidden secrets
|
12
|
+
HIDDEN_PASSWORD='🔑'
|
13
|
+
private_constant :HIDDEN_PASSWORD
|
10
14
|
include Singleton
|
11
15
|
# class methods
|
12
16
|
class << self
|
@@ -23,24 +27,24 @@ module Aspera
|
|
23
27
|
# @param name string or symbol
|
24
28
|
# @param format either pp or json format
|
25
29
|
def dump(name,object,format=:json)
|
26
|
-
|
27
|
-
result=
|
30
|
+
log.debug() do
|
31
|
+
result=
|
32
|
+
case format
|
28
33
|
when :json
|
29
34
|
JSON.pretty_generate(object) rescue PP.pp(object,'')
|
30
35
|
when :ruby
|
31
36
|
PP.pp(object,'')
|
32
37
|
else
|
33
|
-
raise
|
38
|
+
raise 'wrong parameter, expect pp or json'
|
34
39
|
end
|
35
40
|
"#{name.to_s.green} (#{format})=\n#{result}"
|
36
41
|
end
|
37
42
|
end
|
38
|
-
end
|
43
|
+
end # class
|
39
44
|
|
40
|
-
attr_reader :logger_type
|
41
|
-
attr_reader :logger
|
45
|
+
attr_reader :logger_type, :logger
|
42
46
|
attr_writer :program_name
|
43
|
-
attr_accessor :
|
47
|
+
attr_accessor :log_secrets
|
44
48
|
|
45
49
|
# set log level of underlying logger given symbol level
|
46
50
|
def level=(new_level)
|
@@ -53,19 +57,19 @@ module Aspera
|
|
53
57
|
return name.downcase.to_sym if @logger.level.eql?(Logger::Severity.const_get(name))
|
54
58
|
end
|
55
59
|
# should not happen
|
56
|
-
raise "
|
60
|
+
raise "INTERNAL ERROR: unexpected level #{@logger.level}"
|
57
61
|
end
|
58
62
|
|
59
63
|
# change underlying logger, but keep log level
|
60
64
|
def logger_type=(new_logtype)
|
61
65
|
current_severity_integer=@logger.level unless @logger.nil?
|
62
|
-
current_severity_integer=ENV['AS_LOG_LEVEL'] if current_severity_integer.nil?
|
66
|
+
current_severity_integer=ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.has_key?('AS_LOG_LEVEL')
|
63
67
|
current_severity_integer=Logger::Severity::WARN if current_severity_integer.nil?
|
64
68
|
case new_logtype
|
65
69
|
when :stderr
|
66
|
-
@logger = Logger.new(
|
70
|
+
@logger = Logger.new($stderr)
|
67
71
|
when :stdout
|
68
|
-
@logger = Logger.new(
|
72
|
+
@logger = Logger.new($stdout)
|
69
73
|
when :syslog
|
70
74
|
require 'syslog/logger'
|
71
75
|
@logger = Syslog::Logger.new(@program_name)
|
@@ -75,12 +79,14 @@ module Aspera
|
|
75
79
|
@logger.level=current_severity_integer
|
76
80
|
@logger_type=new_logtype
|
77
81
|
original_formatter = @logger.formatter || Logger::Formatter.new
|
78
|
-
# update formatter with password hiding
|
79
|
-
@logger.formatter=
|
80
|
-
|
81
|
-
msg=msg
|
82
|
-
|
83
|
-
|
82
|
+
# update formatter with password hiding, note that @log_secrets may be set AFTER this init is done, so it's done at runtime
|
83
|
+
@logger.formatter=lambda do |severity, datetime, progname, msg|
|
84
|
+
if msg.is_a?(String) && !@log_secrets
|
85
|
+
msg=msg
|
86
|
+
.gsub(/(["':][^"]*(password|secret|private_key)[^"]*["']?[=>: ]+")([^"]+)(")/){"#{Regexp.last_match(1)}#{HIDDEN_PASSWORD}#{Regexp.last_match(4)}"}
|
87
|
+
.gsub(/("[^"]*(secret)[^"]*"=>{)([^}]+)(})/){"#{Regexp.last_match(1)}#{HIDDEN_PASSWORD}#{Regexp.last_match(4)}"}
|
88
|
+
.gsub(/((secrets)={)([^}]+)(})/){"#{Regexp.last_match(1)}#{HIDDEN_PASSWORD}#{Regexp.last_match(4)}"}
|
89
|
+
.gsub(/--+BEGIN[A-Z ]+KEY--+.+--+END[A-Z ]+KEY--+/m){HIDDEN_PASSWORD}
|
84
90
|
end
|
85
91
|
original_formatter.call(severity, datetime, progname, msg)
|
86
92
|
end
|
@@ -91,11 +97,10 @@ module Aspera
|
|
91
97
|
def initialize
|
92
98
|
@logger=nil
|
93
99
|
@program_name='aspera'
|
94
|
-
@
|
100
|
+
@log_secrets=false
|
95
101
|
# this sets @logger and @logger_type (self needed to call method instead of local var)
|
96
102
|
self.logger_type=:stderr
|
97
|
-
raise
|
103
|
+
raise 'error logger shall be defined' if @logger.nil?
|
98
104
|
end
|
99
|
-
|
100
105
|
end
|
101
106
|
end
|
data/lib/aspera/nagios.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'date'
|
2
3
|
|
3
4
|
module Aspera
|
@@ -5,24 +6,24 @@ module Aspera
|
|
5
6
|
# nagios levels
|
6
7
|
LEVELS=[:ok,:warning,:critical,:unknown,:dependent]
|
7
8
|
ADD_PREFIX='add_'
|
8
|
-
# add methods to add nagios error levels, each take component name and message
|
9
|
-
LEVELS.each_index do |code|
|
10
|
-
name="#{ADD_PREFIX}#{LEVELS[code]}".to_sym
|
11
|
-
define_method(name){|comp,msg|@data.push({:code=>code,:comp=>comp,:msg=>msg})}
|
12
|
-
public name
|
13
|
-
end
|
14
9
|
# date offset levels
|
15
10
|
DATE_WARN_OFFSET=2
|
16
11
|
DATE_CRIT_OFFSET=5
|
17
12
|
private_constant :LEVELS,:ADD_PREFIX,:DATE_WARN_OFFSET,:DATE_CRIT_OFFSET
|
18
13
|
|
14
|
+
# add methods to add nagios error levels, each take component name and message
|
15
|
+
LEVELS.each_index do |code|
|
16
|
+
name="#{ADD_PREFIX}#{LEVELS[code]}".to_sym
|
17
|
+
define_method(name){|comp,msg|@data.push({code: code,comp: comp,msg: msg})}
|
18
|
+
end
|
19
|
+
|
19
20
|
attr_reader :data
|
20
21
|
def initialize
|
21
22
|
@data=[]
|
22
23
|
end
|
23
24
|
|
24
25
|
# comparte remote time with local time
|
25
|
-
def check_time_offset(
|
26
|
+
def check_time_offset(remote_date, component)
|
26
27
|
# check date if specified : 2015-10-13T07:32:01Z
|
27
28
|
rtime = DateTime.strptime(remote_date)
|
28
29
|
diff_time = (rtime - DateTime.now).abs
|
@@ -38,22 +39,22 @@ module Aspera
|
|
38
39
|
end
|
39
40
|
end
|
40
41
|
|
41
|
-
def check_product_version(
|
42
|
+
def check_product_version(component, _product, version)
|
42
43
|
add_ok(component,"version #{version}")
|
43
44
|
# TODO check on database if latest version
|
44
45
|
end
|
45
46
|
|
46
47
|
# translate for display
|
47
48
|
def result
|
48
|
-
raise
|
49
|
-
{:
|
49
|
+
raise 'missing result' if @data.empty?
|
50
|
+
{type: :object_list,data: @data.map{|i|{'status'=>LEVELS[i[:code]].to_s,'component'=>i[:comp],'message'=>i[:msg]}}}
|
50
51
|
end
|
51
52
|
|
52
53
|
# process results of a analysis and display status and exit with code
|
53
54
|
def self.process(data)
|
54
|
-
raise
|
55
|
+
raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
|
55
56
|
['status','component','message'].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.has_key?(c)}
|
56
|
-
res_errors = data.
|
57
|
+
res_errors = data.reject{|s|s['status'].eql?('ok')}
|
57
58
|
# keep only errors in case of problem, other ok are assumed so
|
58
59
|
data = res_errors unless res_errors.empty?
|
59
60
|
# first is most critical
|
data/lib/aspera/node.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'aspera/fasp/transfer_spec'
|
2
3
|
require 'aspera/rest'
|
3
4
|
require 'aspera/oauth'
|
4
5
|
require 'aspera/log'
|
@@ -9,33 +10,40 @@ module Aspera
|
|
9
10
|
# Provides additional functions using node API.
|
10
11
|
class Node < Rest
|
11
12
|
# permissions
|
12
|
-
ACCESS_LEVELS
|
13
|
+
ACCESS_LEVELS=%w[delete list mkdir preview read rename write].freeze
|
14
|
+
# prefix for ruby code for filter
|
13
15
|
MATCH_EXEC_PREFIX='exec:'
|
14
16
|
|
15
17
|
# register node special token decoder
|
16
18
|
Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
|
17
19
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
20
|
+
class<<self
|
21
|
+
def set_ak_basic_token(ts,ak,secret)
|
22
|
+
Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
|
23
|
+
ts['token']="Basic #{Base64.strict_encode64("#{ak}:#{secret}")}"
|
24
|
+
end
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
# if prefix: ruby code
|
26
|
-
# if filder is nil, then always match
|
27
|
-
def self.file_matcher(match_expression)
|
28
|
-
match_expression||="#{MATCH_EXEC_PREFIX}true"
|
29
|
-
if match_expression.start_with?(MATCH_EXEC_PREFIX)
|
30
|
-
return eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}")
|
26
|
+
def empty_binding
|
27
|
+
return Kernel.binding
|
31
28
|
end
|
32
|
-
return lambda{|f|f['name'].match(/#{match_expression}/)}
|
33
|
-
end
|
34
29
|
|
35
|
-
|
36
|
-
|
30
|
+
# for access keys: provide expression to match entry in folder
|
31
|
+
# if no prefix: regex
|
32
|
+
# if prefix: ruby code
|
33
|
+
# if filder is nil, then always match
|
34
|
+
def file_matcher(match_expression)
|
35
|
+
match_expression||="#{MATCH_EXEC_PREFIX}true"
|
36
|
+
if match_expression.start_with?(MATCH_EXEC_PREFIX)
|
37
|
+
return eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}", empty_binding, __FILE__, __LINE__)
|
38
|
+
end
|
39
|
+
return lambda{|f|f['name'].match(/#{match_expression}/)}
|
40
|
+
end
|
37
41
|
end
|
38
42
|
|
43
|
+
# def initialize(rest_params)
|
44
|
+
# super(rest_params)
|
45
|
+
# end
|
46
|
+
|
39
47
|
# recursively crawl in a folder.
|
40
48
|
# subfolders a processed if the processing method returns true
|
41
49
|
# @param processor must provide a method to process each entry
|
@@ -54,13 +62,13 @@ module Aspera
|
|
54
62
|
#top_info=read("files/#{opt[:top_file_id]}")[:data]
|
55
63
|
folders_to_explore=[{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
|
56
64
|
Log.dump(:folders_to_explore,folders_to_explore)
|
57
|
-
while !folders_to_explore.empty?
|
65
|
+
while !folders_to_explore.empty?
|
58
66
|
current_item = folders_to_explore.shift
|
59
67
|
Log.log.debug("searching #{current_item[:relpath]}".bg_green)
|
60
68
|
# get folder content
|
61
69
|
folder_contents = begin
|
62
70
|
read("files/#{current_item[:id]}/files")[:data]
|
63
|
-
rescue => e
|
71
|
+
rescue StandardError => e
|
64
72
|
Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
|
65
73
|
[]
|
66
74
|
end
|
@@ -69,8 +77,8 @@ module Aspera
|
|
69
77
|
relative_path=File.join(current_item[:relpath],entry['name'])
|
70
78
|
Log.log.debug("looking #{relative_path}".bg_green)
|
71
79
|
# entry type is file, folder or link
|
72
|
-
if processor.send(opt[:method],entry,relative_path)
|
73
|
-
folders_to_explore.push({:
|
80
|
+
if processor.send(opt[:method],entry,relative_path) && entry['type'].eql?('folder')
|
81
|
+
folders_to_explore.push({id: entry['id'],relpath: relative_path})
|
74
82
|
end
|
75
83
|
end
|
76
84
|
end
|