aspera-cli 4.6.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
@@ -1,4 +1,5 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'aspera/fasp/transfer_spec'
|
2
3
|
require 'aspera/cli/listener/logger'
|
3
4
|
require 'aspera/cli/listener/progress_multi'
|
4
5
|
|
@@ -12,15 +13,15 @@ module Aspera
|
|
12
13
|
FILE_LIST_FROM_ARGS='@args'
|
13
14
|
# special value for --sources : read file list from transfer spec (--ts)
|
14
15
|
FILE_LIST_FROM_TRANSFER_SPEC='@ts'
|
15
|
-
DEFAULT_TRANSFER_NOTIF_TMPL
|
16
|
-
From: <%=from_name%> <<%=from_email%>>
|
17
|
-
To: <<%=to%>>
|
18
|
-
Subject: <%=subject%>
|
16
|
+
DEFAULT_TRANSFER_NOTIF_TMPL=<<~END_OF_TEMPLATE
|
17
|
+
From: <%=from_name%> <<%=from_email%>>
|
18
|
+
To: <<%=to%>>
|
19
|
+
Subject: <%=subject%>
|
19
20
|
|
20
|
-
Transfer is: <%=global_transfer_status%>
|
21
|
+
Transfer is: <%=global_transfer_status%>
|
21
22
|
|
22
|
-
<%=ts.to_yaml%>
|
23
|
-
END_OF_TEMPLATE
|
23
|
+
<%=ts.to_yaml%>
|
24
|
+
END_OF_TEMPLATE
|
24
25
|
#% (formating bug in eclipse)
|
25
26
|
private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
|
26
27
|
TRANSFER_AGENTS=[:direct,:node,:connect,:httpgw,:trsdk]
|
@@ -39,12 +40,12 @@ END_OF_TEMPLATE
|
|
39
40
|
@opt_mgr.set_obj_attr(:ts,self,:option_transfer_spec)
|
40
41
|
@opt_mgr.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:ts,:optional)}")
|
41
42
|
@opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
|
42
|
-
@opt_mgr.add_opt_simple(:to_folder,
|
43
|
-
@opt_mgr.add_opt_simple(:sources,
|
44
|
-
@opt_mgr.add_opt_simple(:transfer_info,
|
45
|
-
@opt_mgr.add_opt_list(:src_type,[:list,:pair],
|
46
|
-
@opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,
|
47
|
-
@opt_mgr.add_opt_list(:progress,[:none,:native,:multi],
|
43
|
+
@opt_mgr.add_opt_simple(:to_folder,'destination folder for downloaded files')
|
44
|
+
@opt_mgr.add_opt_simple(:sources,'list of source files (see doc)')
|
45
|
+
@opt_mgr.add_opt_simple(:transfer_info,'parameters for transfer agent')
|
46
|
+
@opt_mgr.add_opt_list(:src_type,[:list,:pair],'type of file list')
|
47
|
+
@opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,'type of transfer agent')
|
48
|
+
@opt_mgr.add_opt_list(:progress,[:none,:native,:multi],'type of progress bar')
|
48
49
|
@opt_mgr.set_option(:transfer,:direct)
|
49
50
|
@opt_mgr.set_option(:src_type,:list)
|
50
51
|
@opt_mgr.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
|
@@ -58,12 +59,12 @@ END_OF_TEMPLATE
|
|
58
59
|
|
59
60
|
def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
|
60
61
|
|
61
|
-
def
|
62
|
+
def agent_instance=(instance)
|
62
63
|
@agent=instance
|
63
64
|
@agent.add_listener(Listener::Logger.new)
|
64
65
|
# use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
|
65
|
-
if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi)
|
66
|
-
(@opt_mgr.get_option(:progress,:mandatory).eql?(:native)
|
66
|
+
if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) ||
|
67
|
+
(@opt_mgr.get_option(:progress,:mandatory).eql?(:native) && !instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
|
67
68
|
@agent.add_listener(@progress_listener)
|
68
69
|
end
|
69
70
|
end
|
@@ -76,20 +77,20 @@ END_OF_TEMPLATE
|
|
76
77
|
agent_options=@opt_mgr.get_option(:transfer_info,:optional)
|
77
78
|
raise CliBadArgument,"the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), use either @json:<json> or @preset:<parameter set name>" unless [Hash,NilClass].include?(agent_options.class)
|
78
79
|
# special case
|
79
|
-
if agent_type.eql?(:node)
|
80
|
+
if agent_type.eql?(:node) && agent_options.nil?
|
80
81
|
param_set_name=@config.get_plugin_default_config_name(:node)
|
81
82
|
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
82
83
|
agent_options=@config.preset_by_name(param_set_name)
|
83
84
|
end
|
84
85
|
# special case
|
85
|
-
if agent_type.eql?(:direct)
|
86
|
+
if agent_type.eql?(:direct) && @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
|
86
87
|
agent_options={} if agent_options.nil?
|
87
88
|
agent_options[:quiet]=false
|
88
89
|
end
|
89
90
|
agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
|
90
91
|
# get agent instance
|
91
92
|
new_agent=Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
|
92
|
-
|
93
|
+
self.agent_instance=new_agent
|
93
94
|
return nil
|
94
95
|
end
|
95
96
|
|
@@ -103,8 +104,8 @@ END_OF_TEMPLATE
|
|
103
104
|
return dest_folder unless dest_folder.nil?
|
104
105
|
# default: / on remote, . on local
|
105
106
|
case direction.to_s
|
106
|
-
when
|
107
|
-
when
|
107
|
+
when Fasp::TransferSpec::DIRECTION_SEND then dest_folder='/'
|
108
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE then dest_folder='.'
|
108
109
|
else raise "wrong direction: #{direction}"
|
109
110
|
end
|
110
111
|
return dest_folder
|
@@ -112,7 +113,7 @@ END_OF_TEMPLATE
|
|
112
113
|
|
113
114
|
# This is how the list of files to be transfered is specified
|
114
115
|
# get paths suitable for transfer spec from command line
|
115
|
-
# @return {:
|
116
|
+
# @return {source: (mandatory), destination: (optional)}
|
116
117
|
# computation is done only once, cache is kept in @transfer_paths
|
117
118
|
def ts_source_paths
|
118
119
|
# return cache if set
|
@@ -123,25 +124,25 @@ END_OF_TEMPLATE
|
|
123
124
|
file_list=@opt_mgr.get_option(:sources,:optional)
|
124
125
|
case file_list
|
125
126
|
when nil,FILE_LIST_FROM_ARGS
|
126
|
-
Log.log.debug(
|
127
|
+
Log.log.debug('getting file list as parameters')
|
127
128
|
# get remaining arguments
|
128
|
-
file_list=@opt_mgr.get_next_argument(
|
129
|
-
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array)
|
129
|
+
file_list=@opt_mgr.get_next_argument('source file list',:multiple)
|
130
|
+
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) || file_list.empty?
|
130
131
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
131
|
-
Log.log.debug(
|
132
|
+
Log.log.debug('assume list provided in transfer spec')
|
132
133
|
special_case_direct_with_list=@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
|
133
|
-
raise CliBadArgument,
|
134
|
+
raise CliBadArgument,'transfer spec on command line must have sources' if @transfer_paths.nil? && !special_case_direct_with_list
|
134
135
|
# here we assume check of sources is made in transfer agent
|
135
136
|
return @transfer_paths
|
136
137
|
when Array
|
137
|
-
Log.log.debug(
|
138
|
-
raise CliBadArgument,
|
138
|
+
Log.log.debug('getting file list as extended value')
|
139
|
+
raise CliBadArgument,'sources must be a Array of String' if !file_list.reject{|f|f.is_a?(String)}.empty?
|
139
140
|
else
|
140
141
|
raise CliBadArgument,"sources must be a Array, not #{file_list.class}"
|
141
142
|
end
|
142
143
|
# here, file_list is an Array or String
|
143
144
|
if !@transfer_paths.nil?
|
144
|
-
Log.log.warn(
|
145
|
+
Log.log.warn('--sources overrides paths from --ts')
|
145
146
|
end
|
146
147
|
case @opt_mgr.get_option(:src_type,:mandatory)
|
147
148
|
when :list
|
@@ -150,7 +151,7 @@ END_OF_TEMPLATE
|
|
150
151
|
when :pair
|
151
152
|
raise CliBadArgument,"When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
|
152
153
|
@transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
|
153
|
-
else raise
|
154
|
+
else raise 'Unsupported src_type'
|
154
155
|
end
|
155
156
|
Log.log.debug("paths=#{@transfer_paths}")
|
156
157
|
return @transfer_paths
|
@@ -163,14 +164,14 @@ END_OF_TEMPLATE
|
|
163
164
|
# other options are carried to specific agent
|
164
165
|
def start(transfer_spec,tr_opts)
|
165
166
|
# check parameters
|
166
|
-
raise
|
167
|
-
raise
|
167
|
+
raise 'transfer_spec must be hash' unless transfer_spec.is_a?(Hash)
|
168
|
+
raise 'tr_opts must be hash' unless tr_opts.is_a?(Hash)
|
168
169
|
# process :src option
|
169
170
|
case transfer_spec['direction']
|
170
|
-
when
|
171
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE
|
171
172
|
# init default if required in any case
|
172
173
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
173
|
-
when
|
174
|
+
when Fasp::TransferSpec::DIRECTION_SEND
|
174
175
|
case tr_opts[:src]
|
175
176
|
when :direct
|
176
177
|
# init default if required
|
@@ -194,9 +195,10 @@ END_OF_TEMPLATE
|
|
194
195
|
|
195
196
|
transfer_spec.merge!(@transfer_spec_cmdline)
|
196
197
|
# create transfer agent
|
197
|
-
|
198
|
+
set_agent_by_options
|
198
199
|
Log.log.debug("transfer agent is a #{@agent.class}")
|
199
200
|
@agent.start_transfer(transfer_spec,tr_opts)
|
201
|
+
# list of : :success or error message
|
200
202
|
result=@agent.wait_for_transfers_completion
|
201
203
|
@progress_listener.reset
|
202
204
|
Fasp::AgentBase.validate_status_list(result)
|
@@ -219,7 +221,7 @@ END_OF_TEMPLATE
|
|
219
221
|
# @return :success if all sessions statuses returned by "start" are success
|
220
222
|
# else return the first error exception object
|
221
223
|
def self.session_status(statuses)
|
222
|
-
error_statuses=statuses.
|
224
|
+
error_statuses=statuses.reject{|i|i.eql?(:success)}
|
223
225
|
return :success if error_statuses.empty?
|
224
226
|
return error_statuses.first
|
225
227
|
end
|
@@ -228,7 +230,6 @@ END_OF_TEMPLATE
|
|
228
230
|
def shutdown
|
229
231
|
@agent.shutdown if @agent.respond_to?(:shutdown)
|
230
232
|
end
|
231
|
-
|
232
233
|
end
|
233
234
|
end
|
234
235
|
end
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/colors.rb
CHANGED
@@ -1,43 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# simple vt100 colors
|
2
3
|
class String
|
3
|
-
|
4
|
-
|
4
|
+
class<<self
|
5
|
+
private
|
6
|
+
|
7
|
+
def vtcmd(code);"\e[#{code}m";end
|
8
|
+
end
|
5
9
|
# see https://en.wikipedia.org/wiki/ANSI_escape_code
|
6
10
|
# symbol is the method name added to String
|
7
11
|
# it adds control chars to set color (and reset at the end).
|
8
12
|
VTSTYLES = {
|
9
|
-
:
|
10
|
-
:
|
11
|
-
:
|
12
|
-
:
|
13
|
-
:
|
14
|
-
:
|
15
|
-
:
|
16
|
-
:
|
17
|
-
:
|
18
|
-
:
|
19
|
-
:
|
20
|
-
:
|
21
|
-
:
|
22
|
-
:
|
23
|
-
:
|
24
|
-
:
|
25
|
-
:
|
26
|
-
:
|
27
|
-
:
|
28
|
-
:
|
29
|
-
:
|
13
|
+
bold: 1,
|
14
|
+
italic: 3,
|
15
|
+
underline: 4,
|
16
|
+
blink: 5,
|
17
|
+
reverse_color: 7,
|
18
|
+
black: 30,
|
19
|
+
red: 31,
|
20
|
+
green: 32,
|
21
|
+
brown: 33,
|
22
|
+
blue: 34,
|
23
|
+
magenta: 35,
|
24
|
+
cyan: 36,
|
25
|
+
gray: 37,
|
26
|
+
bg_black: 40,
|
27
|
+
bg_red: 41,
|
28
|
+
bg_green: 42,
|
29
|
+
bg_brown: 43,
|
30
|
+
bg_blue: 44,
|
31
|
+
bg_magenta: 45,
|
32
|
+
bg_cyan: 46,
|
33
|
+
bg_gray: 47
|
30
34
|
}
|
31
35
|
private_constant :VTSTYLES
|
32
36
|
# defines methods to String, one per entry in VTSTYLES
|
33
37
|
VTSTYLES.each do |name,code|
|
34
|
-
|
35
|
-
|
36
|
-
|
38
|
+
if $stderr.tty?
|
39
|
+
begin_seq=vtcmd(code)
|
40
|
+
end_code=
|
41
|
+
if code >= 10 then 0
|
42
|
+
elsif code.eql?(1) then 22
|
43
|
+
else code+20
|
44
|
+
end
|
45
|
+
end_seq=vtcmd(end_code)
|
37
46
|
define_method(name){"#{begin_seq}#{self}#{end_seq}"}
|
38
47
|
else
|
39
48
|
define_method(name){self}
|
40
49
|
end
|
41
|
-
public name
|
42
50
|
end
|
43
51
|
end
|
@@ -1,14 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module Aspera
|
2
3
|
# helper class to build command line from a parameter list (key-value hash)
|
3
4
|
# constructor takes hash: { 'param1':'value1', ...}
|
4
5
|
# process_param is called repeatedly with all known parameters
|
5
6
|
# add_env_args is called to get resulting param list and env var (also checks that all params were used)
|
6
7
|
class CommandLineBuilder
|
7
|
-
# transform yes/no to
|
8
|
+
# transform yes/no to true/false
|
8
9
|
def self.yes_to_true(value)
|
9
10
|
case value
|
10
|
-
when 'yes'
|
11
|
-
when 'no'
|
11
|
+
when 'yes' then return true
|
12
|
+
when 'no' then return false
|
12
13
|
end
|
13
14
|
raise "unsupported value: #{value}"
|
14
15
|
end
|
@@ -22,13 +23,13 @@ module Aspera
|
|
22
23
|
options[:mandatory]||=false
|
23
24
|
options[:desc]||=''
|
24
25
|
# by default : string, unless it's without arg
|
25
|
-
if !
|
26
|
+
if !options.has_key?(:accepted_types)
|
26
27
|
options[:accepted_types]=options[:cltype].eql?(:opt_without_arg) ? :bool : :string
|
27
28
|
end
|
28
29
|
# single type is placed in array
|
29
30
|
options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
|
30
|
-
if !options.has_key?(:
|
31
|
-
options[:
|
31
|
+
if !options.has_key?(:clswitch) && options.has_key?(:cltype) && [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
|
32
|
+
options[:clswitch]='--'+param_name.to_s.gsub('_','-')
|
32
33
|
end
|
33
34
|
end
|
34
35
|
end
|
@@ -36,7 +37,7 @@ module Aspera
|
|
36
37
|
private
|
37
38
|
|
38
39
|
# clvarname : command line variable name
|
39
|
-
def env_name(
|
40
|
+
def env_name(_param_name,options)
|
40
41
|
return options[:clvarname]
|
41
42
|
end
|
42
43
|
|
@@ -46,7 +47,7 @@ module Aspera
|
|
46
47
|
|
47
48
|
# @param param_hash
|
48
49
|
def initialize(param_hash,params_definition)
|
49
|
-
@param_hash=param_hash
|
50
|
+
@param_hash=param_hash # keep reference so that it can be modified by caller before calling `process_params`
|
50
51
|
@params_definition=params_definition
|
51
52
|
@result_env={}
|
52
53
|
@result_args=[]
|
@@ -86,42 +87,53 @@ module Aspera
|
|
86
87
|
# @param options : options for type
|
87
88
|
def process_param(param_name,action=nil)
|
88
89
|
options=@params_definition[param_name]
|
89
|
-
action=options[:cltype] if action.nil?
|
90
90
|
# should not happen
|
91
|
-
|
91
|
+
if options.nil?
|
92
|
+
Log.log.warn("Unknown parameter #{param_name}")
|
93
|
+
return
|
94
|
+
end
|
95
|
+
action=options[:cltype] if action.nil?
|
92
96
|
# check mandatory parameter (nil is valid value)
|
93
|
-
raise Fasp::Error
|
97
|
+
raise Fasp::Error, "Missing mandatory parameter: #{param_name}" if options[:mandatory] && !@param_hash.has_key?(param_name)
|
94
98
|
parameter_value=@param_hash[param_name]
|
99
|
+
|
95
100
|
#parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
|
101
|
+
|
102
|
+
# Check parameter type
|
96
103
|
expected_classes=options[:accepted_types].map do |s|
|
97
104
|
case s
|
98
|
-
when :string
|
99
|
-
when :array
|
100
|
-
when :hash
|
101
|
-
when :int
|
102
|
-
when :bool
|
105
|
+
when :string then String
|
106
|
+
when :array then Array
|
107
|
+
when :hash then Hash
|
108
|
+
when :int then Integer
|
109
|
+
when :bool then [TrueClass,FalseClass]
|
103
110
|
else raise "INTERNAL: unexpected value: #{s}"
|
104
111
|
end
|
105
112
|
end.flatten
|
106
|
-
#
|
107
|
-
raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or expected_classes.include?(parameter_value.class)
|
113
|
+
raise Fasp::Error,"#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, " unless parameter_value.nil? || expected_classes.include?(parameter_value.class)
|
108
114
|
@used_param_names.push(param_name) unless action.eql?(:defer)
|
109
115
|
|
110
116
|
# process only non-nil values
|
111
117
|
return nil if parameter_value.nil?
|
112
118
|
|
113
|
-
|
119
|
+
# check that value is of an accepted type (string, int bool)
|
120
|
+
raise "Value #{parameter_value} is not allowed for #{param_name}" if options.has_key?(:enum) && !options[:enum].include?(parameter_value)
|
121
|
+
|
122
|
+
# convert some values if value on command line needs processing from value in structure
|
123
|
+
case options[:clconvert]
|
124
|
+
when Hash
|
114
125
|
# translate using conversion table
|
115
|
-
new_value=options[:
|
116
|
-
raise "unsupported value: #{parameter_value}" if new_value.nil?
|
126
|
+
new_value=options[:clconvert][parameter_value]
|
127
|
+
raise "unsupported value: #{parameter_value}, expect: #{options[:clconvert].keys.join(', ')}" if new_value.nil?
|
117
128
|
parameter_value=new_value
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
|
129
|
+
when String
|
130
|
+
# :clconvert has name of class and encoding method
|
131
|
+
convclass,convmethod=options[:clconvert].split('.')
|
132
|
+
newvalue=Kernel.const_get(convclass).send(convmethod,parameter_value)
|
133
|
+
raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}" if newvalue.nil?
|
124
134
|
parameter_value=newvalue
|
135
|
+
when NilClass
|
136
|
+
else raise "not expected type for clconvert #{options[:clconvert].class} for #{param_name}"
|
125
137
|
end
|
126
138
|
|
127
139
|
case action
|
@@ -135,19 +147,21 @@ module Aspera
|
|
135
147
|
when :opt_without_arg # if present and true : just add option without value
|
136
148
|
add_param=false
|
137
149
|
case parameter_value
|
138
|
-
when false# nothing to put on command line, no creation by default
|
139
|
-
when true
|
140
|
-
else raise Fasp::Error
|
150
|
+
when false then nil # nothing to put on command line, no creation by default
|
151
|
+
when true then add_param=true
|
152
|
+
else raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}"
|
141
153
|
end
|
142
|
-
add_param
|
143
|
-
add_command_line_options([options[:
|
154
|
+
add_param= !add_param if options[:add_on_false]
|
155
|
+
add_command_line_options([options[:clswitch]]) if add_param
|
144
156
|
when :opt_with_arg # transform into command line option with value
|
145
157
|
#parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
|
146
158
|
parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
|
147
159
|
# if transfer_spec value is an array, applies option many times
|
148
|
-
parameter_value.each{|v|add_command_line_options([options[:
|
160
|
+
parameter_value.each{|v|add_command_line_options([options[:clswitch],v])}
|
161
|
+
when NilClass
|
162
|
+
Log.log.debug("Ignoring parameter: #{param_name}")
|
149
163
|
else
|
150
|
-
raise "
|
164
|
+
raise "ERROR: unknown action: #{action}/#{action.class}"
|
151
165
|
end
|
152
166
|
end
|
153
167
|
end
|
data/lib/aspera/cos_node.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/log'
|
2
3
|
require 'aspera/rest'
|
3
4
|
require 'xmlsimple'
|
@@ -11,25 +12,28 @@ module Aspera
|
|
11
12
|
@auth_url=auth_url
|
12
13
|
@api_key=api_key
|
13
14
|
s3_api=Aspera::Rest.new({
|
14
|
-
:
|
15
|
-
:
|
16
|
-
:
|
17
|
-
:
|
18
|
-
|
19
|
-
:
|
20
|
-
:
|
21
|
-
:
|
22
|
-
|
15
|
+
base_url: storage_endpoint,
|
16
|
+
not_auth_codes: ['401','403'], # error codes when not authorized
|
17
|
+
headers: {'ibm-service-instance-id' => instance_id},
|
18
|
+
auth: {
|
19
|
+
type: :oauth2,
|
20
|
+
base_url: @auth_url,
|
21
|
+
crtype: :generic,
|
22
|
+
generic: {
|
23
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
24
|
+
response_type: 'cloud_iam',
|
25
|
+
apikey: @api_key
|
26
|
+
}}})
|
23
27
|
# read FASP connection information for bucket
|
24
|
-
xml_result_text=s3_api.call({:
|
28
|
+
xml_result_text=s3_api.call({operation: 'GET',subpath: bucket_name,headers: {'Accept'=>'application/xml'},url_params: {'faspConnectionInfo'=>nil}})[:http].body
|
25
29
|
ats_info=XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
|
26
30
|
Aspera::Log.dump('ats_info',ats_info)
|
27
31
|
super({
|
28
|
-
:
|
29
|
-
:
|
30
|
-
|
31
|
-
:
|
32
|
-
:
|
32
|
+
base_url: ats_info['ATSEndpoint'],
|
33
|
+
auth: {
|
34
|
+
type: :basic,
|
35
|
+
username: ats_info['AccessKey']['Id'],
|
36
|
+
password: ats_info['AccessKey']['Secret']}})
|
33
37
|
# prepare transfer spec addition
|
34
38
|
@add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
|
35
39
|
'type' => 'token',
|
@@ -42,12 +46,16 @@ module Aspera
|
|
42
46
|
def generate_token
|
43
47
|
# OAuth API to get delegated token
|
44
48
|
delegated_oauth=Oauth.new({
|
45
|
-
|
46
|
-
:
|
47
|
-
:
|
48
|
-
:
|
49
|
-
:
|
50
|
-
|
49
|
+
type: :oauth2,
|
50
|
+
base_url: @auth_url,
|
51
|
+
token_field: TOKEN_FIELD,
|
52
|
+
crtype: :generic,
|
53
|
+
generic: {
|
54
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
55
|
+
response_type: 'delegated_refresh_token',
|
56
|
+
apikey: @api_key,
|
57
|
+
receiver_client_ids: 'aspera_ats'
|
58
|
+
}})
|
51
59
|
# get delagated token to be placed in rest call header and in transfer tags
|
52
60
|
@add_ts['tags']['aspera']['node']['storage_credentials']['token'][TOKEN_FIELD]=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
|
53
61
|
@params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/log'
|
2
3
|
require 'singleton'
|
3
4
|
|
@@ -6,8 +7,8 @@ module Aspera
|
|
6
7
|
class DataRepository
|
7
8
|
include Singleton
|
8
9
|
# get binary value from data repository
|
9
|
-
def
|
10
|
-
File.read(File.join(
|
10
|
+
def data(id)
|
11
|
+
File.read(File.join(__dir__,'data',id.to_s),mode: 'rb')
|
11
12
|
end
|
12
13
|
end
|
13
14
|
end
|
data/lib/aspera/environment.rb
CHANGED
@@ -1,66 +1,71 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/log'
|
2
3
|
require 'rbconfig'
|
3
4
|
|
4
5
|
module Aspera
|
5
|
-
# detect OS, architecture, and
|
6
|
+
# detect OS, architecture, and specific stuff
|
6
7
|
class Environment
|
7
8
|
OS_WINDOWS = :windows
|
8
9
|
OS_X = :osx
|
9
10
|
OS_LINUX = :linux
|
10
11
|
OS_AIX = :aix
|
11
|
-
OS_LIST=[OS_WINDOWS,OS_X,OS_LINUX,OS_AIX]
|
12
|
-
|
13
|
-
def self.os
|
14
|
-
case RbConfig::CONFIG['host_os']
|
15
|
-
when /mswin/,/msys/,/mingw/,/cygwin/,/bccwin/,/wince/,/emc/
|
16
|
-
return OS_WINDOWS
|
17
|
-
when /darwin/,/mac os/
|
18
|
-
return OS_X
|
19
|
-
when /linux/
|
20
|
-
return OS_LINUX
|
21
|
-
when /aix/
|
22
|
-
return OS_AIX
|
23
|
-
else
|
24
|
-
raise "Unknown OS: #{RbConfig::CONFIG['host_os']}"
|
25
|
-
end
|
26
|
-
end
|
27
|
-
CPU_X86_64=:x86_64
|
12
|
+
OS_LIST=[OS_WINDOWS,OS_X,OS_LINUX,OS_AIX].freeze
|
13
|
+
CPU_X86_64=:x86_64 # rubocop:disable Naming/VariableNumber
|
28
14
|
CPU_PPC64=:ppc64
|
29
15
|
CPU_PPC64LE=:ppc64le
|
30
16
|
CPU_S390=:s390
|
31
|
-
CPU_LIST=[CPU_X86_64,CPU_PPC64,CPU_PPC64LE,CPU_S390]
|
17
|
+
CPU_LIST=[CPU_X86_64,CPU_PPC64,CPU_PPC64LE,CPU_S390].freeze
|
32
18
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
return CPU_X86_64
|
37
|
-
when /powerpc/,/ppc64/
|
38
|
-
return CPU_PPC64LE if os.eql?(OS_LINUX)
|
39
|
-
return CPU_PPC64
|
40
|
-
when /s390/
|
41
|
-
return CPU_S390
|
42
|
-
else # other
|
43
|
-
raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
|
19
|
+
class<<self
|
20
|
+
def ruby_version
|
21
|
+
return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
|
44
22
|
end
|
45
|
-
end
|
46
23
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
24
|
+
def os
|
25
|
+
case RbConfig::CONFIG['host_os']
|
26
|
+
when /mswin/,/msys/,/mingw/,/cygwin/,/bccwin/,/wince/,/emc/
|
27
|
+
return OS_WINDOWS
|
28
|
+
when /darwin/,/mac os/
|
29
|
+
return OS_X
|
30
|
+
when /linux/
|
31
|
+
return OS_LINUX
|
32
|
+
when /aix/
|
33
|
+
return OS_AIX
|
34
|
+
else
|
35
|
+
raise "Unknown OS: #{RbConfig::CONFIG['host_os']}"
|
36
|
+
end
|
37
|
+
end
|
55
38
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
39
|
+
def cpu
|
40
|
+
case RbConfig::CONFIG['host_cpu']
|
41
|
+
when /x86_64/,/x64/
|
42
|
+
return CPU_X86_64
|
43
|
+
when /powerpc/,/ppc64/
|
44
|
+
return CPU_PPC64LE if os.eql?(OS_LINUX)
|
45
|
+
return CPU_PPC64
|
46
|
+
when /s390/
|
47
|
+
return CPU_S390
|
48
|
+
else # other
|
49
|
+
raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
|
62
50
|
end
|
63
51
|
end
|
52
|
+
|
53
|
+
def architecture
|
54
|
+
return "#{os}-#{cpu}"
|
55
|
+
end
|
56
|
+
|
57
|
+
def exe_extension
|
58
|
+
return '.exe' if os.eql?(OS_WINDOWS)
|
59
|
+
return ''
|
60
|
+
end
|
61
|
+
|
62
|
+
# on Windows, the env var %USERPROFILE% provides the path to user's home more reliably than %HOMEDRIVE%%HOMEPATH%
|
63
|
+
# so, tell Ruby the right way
|
64
|
+
def fix_home
|
65
|
+
return unless os.eql?(OS_WINDOWS) && ENV.has_key?('USERPROFILE') && Dir.exist?(ENV['USERPROFILE'])
|
66
|
+
ENV['HOME']=ENV['USERPROFILE']
|
67
|
+
Log.log.debug("Windows: set home to USERPROFILE: #{ENV['HOME']}")
|
68
|
+
end
|
64
69
|
end
|
65
70
|
end
|
66
71
|
end
|