aspera-cli 4.6.0 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'aspera/fasp/transfer_spec'
|
|
2
3
|
require 'aspera/cli/listener/logger'
|
|
3
4
|
require 'aspera/cli/listener/progress_multi'
|
|
4
5
|
|
|
@@ -12,15 +13,15 @@ module Aspera
|
|
|
12
13
|
FILE_LIST_FROM_ARGS='@args'
|
|
13
14
|
# special value for --sources : read file list from transfer spec (--ts)
|
|
14
15
|
FILE_LIST_FROM_TRANSFER_SPEC='@ts'
|
|
15
|
-
DEFAULT_TRANSFER_NOTIF_TMPL
|
|
16
|
-
From: <%=from_name%> <<%=from_email%>>
|
|
17
|
-
To: <<%=to%>>
|
|
18
|
-
Subject: <%=subject%>
|
|
16
|
+
DEFAULT_TRANSFER_NOTIF_TMPL=<<~END_OF_TEMPLATE
|
|
17
|
+
From: <%=from_name%> <<%=from_email%>>
|
|
18
|
+
To: <<%=to%>>
|
|
19
|
+
Subject: <%=subject%>
|
|
19
20
|
|
|
20
|
-
Transfer is: <%=global_transfer_status%>
|
|
21
|
+
Transfer is: <%=global_transfer_status%>
|
|
21
22
|
|
|
22
|
-
<%=ts.to_yaml%>
|
|
23
|
-
END_OF_TEMPLATE
|
|
23
|
+
<%=ts.to_yaml%>
|
|
24
|
+
END_OF_TEMPLATE
|
|
24
25
|
#% (formating bug in eclipse)
|
|
25
26
|
private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
|
|
26
27
|
TRANSFER_AGENTS=[:direct,:node,:connect,:httpgw,:trsdk]
|
|
@@ -39,12 +40,12 @@ END_OF_TEMPLATE
|
|
|
39
40
|
@opt_mgr.set_obj_attr(:ts,self,:option_transfer_spec)
|
|
40
41
|
@opt_mgr.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:ts,:optional)}")
|
|
41
42
|
@opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
|
|
42
|
-
@opt_mgr.add_opt_simple(:to_folder,
|
|
43
|
-
@opt_mgr.add_opt_simple(:sources,
|
|
44
|
-
@opt_mgr.add_opt_simple(:transfer_info,
|
|
45
|
-
@opt_mgr.add_opt_list(:src_type,[:list,:pair],
|
|
46
|
-
@opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,
|
|
47
|
-
@opt_mgr.add_opt_list(:progress,[:none,:native,:multi],
|
|
43
|
+
@opt_mgr.add_opt_simple(:to_folder,'destination folder for downloaded files')
|
|
44
|
+
@opt_mgr.add_opt_simple(:sources,'list of source files (see doc)')
|
|
45
|
+
@opt_mgr.add_opt_simple(:transfer_info,'parameters for transfer agent')
|
|
46
|
+
@opt_mgr.add_opt_list(:src_type,[:list,:pair],'type of file list')
|
|
47
|
+
@opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,'type of transfer agent')
|
|
48
|
+
@opt_mgr.add_opt_list(:progress,[:none,:native,:multi],'type of progress bar')
|
|
48
49
|
@opt_mgr.set_option(:transfer,:direct)
|
|
49
50
|
@opt_mgr.set_option(:src_type,:list)
|
|
50
51
|
@opt_mgr.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
|
|
@@ -58,12 +59,12 @@ END_OF_TEMPLATE
|
|
|
58
59
|
|
|
59
60
|
def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
|
|
60
61
|
|
|
61
|
-
def
|
|
62
|
+
def agent_instance=(instance)
|
|
62
63
|
@agent=instance
|
|
63
64
|
@agent.add_listener(Listener::Logger.new)
|
|
64
65
|
# use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
|
|
65
|
-
if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi)
|
|
66
|
-
(@opt_mgr.get_option(:progress,:mandatory).eql?(:native)
|
|
66
|
+
if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) ||
|
|
67
|
+
(@opt_mgr.get_option(:progress,:mandatory).eql?(:native) && !instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
|
|
67
68
|
@agent.add_listener(@progress_listener)
|
|
68
69
|
end
|
|
69
70
|
end
|
|
@@ -76,20 +77,20 @@ END_OF_TEMPLATE
|
|
|
76
77
|
agent_options=@opt_mgr.get_option(:transfer_info,:optional)
|
|
77
78
|
raise CliBadArgument,"the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), use either @json:<json> or @preset:<parameter set name>" unless [Hash,NilClass].include?(agent_options.class)
|
|
78
79
|
# special case
|
|
79
|
-
if agent_type.eql?(:node)
|
|
80
|
+
if agent_type.eql?(:node) && agent_options.nil?
|
|
80
81
|
param_set_name=@config.get_plugin_default_config_name(:node)
|
|
81
82
|
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
|
82
83
|
agent_options=@config.preset_by_name(param_set_name)
|
|
83
84
|
end
|
|
84
85
|
# special case
|
|
85
|
-
if agent_type.eql?(:direct)
|
|
86
|
+
if agent_type.eql?(:direct) && @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
|
|
86
87
|
agent_options={} if agent_options.nil?
|
|
87
88
|
agent_options[:quiet]=false
|
|
88
89
|
end
|
|
89
90
|
agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
|
|
90
91
|
# get agent instance
|
|
91
92
|
new_agent=Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
|
|
92
|
-
|
|
93
|
+
self.agent_instance=new_agent
|
|
93
94
|
return nil
|
|
94
95
|
end
|
|
95
96
|
|
|
@@ -103,8 +104,8 @@ END_OF_TEMPLATE
|
|
|
103
104
|
return dest_folder unless dest_folder.nil?
|
|
104
105
|
# default: / on remote, . on local
|
|
105
106
|
case direction.to_s
|
|
106
|
-
when
|
|
107
|
-
when
|
|
107
|
+
when Fasp::TransferSpec::DIRECTION_SEND then dest_folder='/'
|
|
108
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE then dest_folder='.'
|
|
108
109
|
else raise "wrong direction: #{direction}"
|
|
109
110
|
end
|
|
110
111
|
return dest_folder
|
|
@@ -112,7 +113,7 @@ END_OF_TEMPLATE
|
|
|
112
113
|
|
|
113
114
|
# This is how the list of files to be transfered is specified
|
|
114
115
|
# get paths suitable for transfer spec from command line
|
|
115
|
-
# @return {:
|
|
116
|
+
# @return {source: (mandatory), destination: (optional)}
|
|
116
117
|
# computation is done only once, cache is kept in @transfer_paths
|
|
117
118
|
def ts_source_paths
|
|
118
119
|
# return cache if set
|
|
@@ -123,25 +124,25 @@ END_OF_TEMPLATE
|
|
|
123
124
|
file_list=@opt_mgr.get_option(:sources,:optional)
|
|
124
125
|
case file_list
|
|
125
126
|
when nil,FILE_LIST_FROM_ARGS
|
|
126
|
-
Log.log.debug(
|
|
127
|
+
Log.log.debug('getting file list as parameters')
|
|
127
128
|
# get remaining arguments
|
|
128
|
-
file_list=@opt_mgr.get_next_argument(
|
|
129
|
-
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array)
|
|
129
|
+
file_list=@opt_mgr.get_next_argument('source file list',:multiple)
|
|
130
|
+
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) || file_list.empty?
|
|
130
131
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
|
131
|
-
Log.log.debug(
|
|
132
|
+
Log.log.debug('assume list provided in transfer spec')
|
|
132
133
|
special_case_direct_with_list=@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
|
|
133
|
-
raise CliBadArgument,
|
|
134
|
+
raise CliBadArgument,'transfer spec on command line must have sources' if @transfer_paths.nil? && !special_case_direct_with_list
|
|
134
135
|
# here we assume check of sources is made in transfer agent
|
|
135
136
|
return @transfer_paths
|
|
136
137
|
when Array
|
|
137
|
-
Log.log.debug(
|
|
138
|
-
raise CliBadArgument,
|
|
138
|
+
Log.log.debug('getting file list as extended value')
|
|
139
|
+
raise CliBadArgument,'sources must be a Array of String' if !file_list.reject{|f|f.is_a?(String)}.empty?
|
|
139
140
|
else
|
|
140
141
|
raise CliBadArgument,"sources must be a Array, not #{file_list.class}"
|
|
141
142
|
end
|
|
142
143
|
# here, file_list is an Array or String
|
|
143
144
|
if !@transfer_paths.nil?
|
|
144
|
-
Log.log.warn(
|
|
145
|
+
Log.log.warn('--sources overrides paths from --ts')
|
|
145
146
|
end
|
|
146
147
|
case @opt_mgr.get_option(:src_type,:mandatory)
|
|
147
148
|
when :list
|
|
@@ -150,7 +151,7 @@ END_OF_TEMPLATE
|
|
|
150
151
|
when :pair
|
|
151
152
|
raise CliBadArgument,"When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
|
|
152
153
|
@transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
|
|
153
|
-
else raise
|
|
154
|
+
else raise 'Unsupported src_type'
|
|
154
155
|
end
|
|
155
156
|
Log.log.debug("paths=#{@transfer_paths}")
|
|
156
157
|
return @transfer_paths
|
|
@@ -163,14 +164,14 @@ END_OF_TEMPLATE
|
|
|
163
164
|
# other options are carried to specific agent
|
|
164
165
|
def start(transfer_spec,tr_opts)
|
|
165
166
|
# check parameters
|
|
166
|
-
raise
|
|
167
|
-
raise
|
|
167
|
+
raise 'transfer_spec must be hash' unless transfer_spec.is_a?(Hash)
|
|
168
|
+
raise 'tr_opts must be hash' unless tr_opts.is_a?(Hash)
|
|
168
169
|
# process :src option
|
|
169
170
|
case transfer_spec['direction']
|
|
170
|
-
when
|
|
171
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE
|
|
171
172
|
# init default if required in any case
|
|
172
173
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
|
173
|
-
when
|
|
174
|
+
when Fasp::TransferSpec::DIRECTION_SEND
|
|
174
175
|
case tr_opts[:src]
|
|
175
176
|
when :direct
|
|
176
177
|
# init default if required
|
|
@@ -194,9 +195,10 @@ END_OF_TEMPLATE
|
|
|
194
195
|
|
|
195
196
|
transfer_spec.merge!(@transfer_spec_cmdline)
|
|
196
197
|
# create transfer agent
|
|
197
|
-
|
|
198
|
+
set_agent_by_options
|
|
198
199
|
Log.log.debug("transfer agent is a #{@agent.class}")
|
|
199
200
|
@agent.start_transfer(transfer_spec,tr_opts)
|
|
201
|
+
# list of : :success or error message
|
|
200
202
|
result=@agent.wait_for_transfers_completion
|
|
201
203
|
@progress_listener.reset
|
|
202
204
|
Fasp::AgentBase.validate_status_list(result)
|
|
@@ -219,7 +221,7 @@ END_OF_TEMPLATE
|
|
|
219
221
|
# @return :success if all sessions statuses returned by "start" are success
|
|
220
222
|
# else return the first error exception object
|
|
221
223
|
def self.session_status(statuses)
|
|
222
|
-
error_statuses=statuses.
|
|
224
|
+
error_statuses=statuses.reject{|i|i.eql?(:success)}
|
|
223
225
|
return :success if error_statuses.empty?
|
|
224
226
|
return error_statuses.first
|
|
225
227
|
end
|
|
@@ -228,7 +230,6 @@ END_OF_TEMPLATE
|
|
|
228
230
|
def shutdown
|
|
229
231
|
@agent.shutdown if @agent.respond_to?(:shutdown)
|
|
230
232
|
end
|
|
231
|
-
|
|
232
233
|
end
|
|
233
234
|
end
|
|
234
235
|
end
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/colors.rb
CHANGED
|
@@ -1,43 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
# simple vt100 colors
|
|
2
3
|
class String
|
|
3
|
-
|
|
4
|
-
|
|
4
|
+
class<<self
|
|
5
|
+
private
|
|
6
|
+
|
|
7
|
+
def vtcmd(code);"\e[#{code}m";end
|
|
8
|
+
end
|
|
5
9
|
# see https://en.wikipedia.org/wiki/ANSI_escape_code
|
|
6
10
|
# symbol is the method name added to String
|
|
7
11
|
# it adds control chars to set color (and reset at the end).
|
|
8
12
|
VTSTYLES = {
|
|
9
|
-
:
|
|
10
|
-
:
|
|
11
|
-
:
|
|
12
|
-
:
|
|
13
|
-
:
|
|
14
|
-
:
|
|
15
|
-
:
|
|
16
|
-
:
|
|
17
|
-
:
|
|
18
|
-
:
|
|
19
|
-
:
|
|
20
|
-
:
|
|
21
|
-
:
|
|
22
|
-
:
|
|
23
|
-
:
|
|
24
|
-
:
|
|
25
|
-
:
|
|
26
|
-
:
|
|
27
|
-
:
|
|
28
|
-
:
|
|
29
|
-
:
|
|
13
|
+
bold: 1,
|
|
14
|
+
italic: 3,
|
|
15
|
+
underline: 4,
|
|
16
|
+
blink: 5,
|
|
17
|
+
reverse_color: 7,
|
|
18
|
+
black: 30,
|
|
19
|
+
red: 31,
|
|
20
|
+
green: 32,
|
|
21
|
+
brown: 33,
|
|
22
|
+
blue: 34,
|
|
23
|
+
magenta: 35,
|
|
24
|
+
cyan: 36,
|
|
25
|
+
gray: 37,
|
|
26
|
+
bg_black: 40,
|
|
27
|
+
bg_red: 41,
|
|
28
|
+
bg_green: 42,
|
|
29
|
+
bg_brown: 43,
|
|
30
|
+
bg_blue: 44,
|
|
31
|
+
bg_magenta: 45,
|
|
32
|
+
bg_cyan: 46,
|
|
33
|
+
bg_gray: 47
|
|
30
34
|
}
|
|
31
35
|
private_constant :VTSTYLES
|
|
32
36
|
# defines methods to String, one per entry in VTSTYLES
|
|
33
37
|
VTSTYLES.each do |name,code|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
38
|
+
if $stderr.tty?
|
|
39
|
+
begin_seq=vtcmd(code)
|
|
40
|
+
end_code=
|
|
41
|
+
if code >= 10 then 0
|
|
42
|
+
elsif code.eql?(1) then 22
|
|
43
|
+
else code+20
|
|
44
|
+
end
|
|
45
|
+
end_seq=vtcmd(end_code)
|
|
37
46
|
define_method(name){"#{begin_seq}#{self}#{end_seq}"}
|
|
38
47
|
else
|
|
39
48
|
define_method(name){self}
|
|
40
49
|
end
|
|
41
|
-
public name
|
|
42
50
|
end
|
|
43
51
|
end
|
|
@@ -1,14 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
module Aspera
|
|
2
3
|
# helper class to build command line from a parameter list (key-value hash)
|
|
3
4
|
# constructor takes hash: { 'param1':'value1', ...}
|
|
4
5
|
# process_param is called repeatedly with all known parameters
|
|
5
6
|
# add_env_args is called to get resulting param list and env var (also checks that all params were used)
|
|
6
7
|
class CommandLineBuilder
|
|
7
|
-
# transform yes/no to
|
|
8
|
+
# transform yes/no to true/false
|
|
8
9
|
def self.yes_to_true(value)
|
|
9
10
|
case value
|
|
10
|
-
when 'yes'
|
|
11
|
-
when 'no'
|
|
11
|
+
when 'yes' then return true
|
|
12
|
+
when 'no' then return false
|
|
12
13
|
end
|
|
13
14
|
raise "unsupported value: #{value}"
|
|
14
15
|
end
|
|
@@ -22,13 +23,13 @@ module Aspera
|
|
|
22
23
|
options[:mandatory]||=false
|
|
23
24
|
options[:desc]||=''
|
|
24
25
|
# by default : string, unless it's without arg
|
|
25
|
-
if !
|
|
26
|
+
if !options.has_key?(:accepted_types)
|
|
26
27
|
options[:accepted_types]=options[:cltype].eql?(:opt_without_arg) ? :bool : :string
|
|
27
28
|
end
|
|
28
29
|
# single type is placed in array
|
|
29
30
|
options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
|
|
30
|
-
if !options.has_key?(:
|
|
31
|
-
options[:
|
|
31
|
+
if !options.has_key?(:clswitch) && options.has_key?(:cltype) && [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
|
|
32
|
+
options[:clswitch]='--'+param_name.to_s.gsub('_','-')
|
|
32
33
|
end
|
|
33
34
|
end
|
|
34
35
|
end
|
|
@@ -36,7 +37,7 @@ module Aspera
|
|
|
36
37
|
private
|
|
37
38
|
|
|
38
39
|
# clvarname : command line variable name
|
|
39
|
-
def env_name(
|
|
40
|
+
def env_name(_param_name,options)
|
|
40
41
|
return options[:clvarname]
|
|
41
42
|
end
|
|
42
43
|
|
|
@@ -46,7 +47,7 @@ module Aspera
|
|
|
46
47
|
|
|
47
48
|
# @param param_hash
|
|
48
49
|
def initialize(param_hash,params_definition)
|
|
49
|
-
@param_hash=param_hash
|
|
50
|
+
@param_hash=param_hash # keep reference so that it can be modified by caller before calling `process_params`
|
|
50
51
|
@params_definition=params_definition
|
|
51
52
|
@result_env={}
|
|
52
53
|
@result_args=[]
|
|
@@ -86,42 +87,53 @@ module Aspera
|
|
|
86
87
|
# @param options : options for type
|
|
87
88
|
def process_param(param_name,action=nil)
|
|
88
89
|
options=@params_definition[param_name]
|
|
89
|
-
action=options[:cltype] if action.nil?
|
|
90
90
|
# should not happen
|
|
91
|
-
|
|
91
|
+
if options.nil?
|
|
92
|
+
Log.log.warn("Unknown parameter #{param_name}")
|
|
93
|
+
return
|
|
94
|
+
end
|
|
95
|
+
action=options[:cltype] if action.nil?
|
|
92
96
|
# check mandatory parameter (nil is valid value)
|
|
93
|
-
raise Fasp::Error
|
|
97
|
+
raise Fasp::Error, "Missing mandatory parameter: #{param_name}" if options[:mandatory] && !@param_hash.has_key?(param_name)
|
|
94
98
|
parameter_value=@param_hash[param_name]
|
|
99
|
+
|
|
95
100
|
#parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
|
|
101
|
+
|
|
102
|
+
# Check parameter type
|
|
96
103
|
expected_classes=options[:accepted_types].map do |s|
|
|
97
104
|
case s
|
|
98
|
-
when :string
|
|
99
|
-
when :array
|
|
100
|
-
when :hash
|
|
101
|
-
when :int
|
|
102
|
-
when :bool
|
|
105
|
+
when :string then String
|
|
106
|
+
when :array then Array
|
|
107
|
+
when :hash then Hash
|
|
108
|
+
when :int then Integer
|
|
109
|
+
when :bool then [TrueClass,FalseClass]
|
|
103
110
|
else raise "INTERNAL: unexpected value: #{s}"
|
|
104
111
|
end
|
|
105
112
|
end.flatten
|
|
106
|
-
#
|
|
107
|
-
raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or expected_classes.include?(parameter_value.class)
|
|
113
|
+
raise Fasp::Error,"#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, " unless parameter_value.nil? || expected_classes.include?(parameter_value.class)
|
|
108
114
|
@used_param_names.push(param_name) unless action.eql?(:defer)
|
|
109
115
|
|
|
110
116
|
# process only non-nil values
|
|
111
117
|
return nil if parameter_value.nil?
|
|
112
118
|
|
|
113
|
-
|
|
119
|
+
# check that value is of an accepted type (string, int bool)
|
|
120
|
+
raise "Value #{parameter_value} is not allowed for #{param_name}" if options.has_key?(:enum) && !options[:enum].include?(parameter_value)
|
|
121
|
+
|
|
122
|
+
# convert some values if value on command line needs processing from value in structure
|
|
123
|
+
case options[:clconvert]
|
|
124
|
+
when Hash
|
|
114
125
|
# translate using conversion table
|
|
115
|
-
new_value=options[:
|
|
116
|
-
raise "unsupported value: #{parameter_value}" if new_value.nil?
|
|
126
|
+
new_value=options[:clconvert][parameter_value]
|
|
127
|
+
raise "unsupported value: #{parameter_value}, expect: #{options[:clconvert].keys.join(', ')}" if new_value.nil?
|
|
117
128
|
parameter_value=new_value
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
|
|
129
|
+
when String
|
|
130
|
+
# :clconvert has name of class and encoding method
|
|
131
|
+
convclass,convmethod=options[:clconvert].split('.')
|
|
132
|
+
newvalue=Kernel.const_get(convclass).send(convmethod,parameter_value)
|
|
133
|
+
raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}" if newvalue.nil?
|
|
124
134
|
parameter_value=newvalue
|
|
135
|
+
when NilClass
|
|
136
|
+
else raise "not expected type for clconvert #{options[:clconvert].class} for #{param_name}"
|
|
125
137
|
end
|
|
126
138
|
|
|
127
139
|
case action
|
|
@@ -135,19 +147,21 @@ module Aspera
|
|
|
135
147
|
when :opt_without_arg # if present and true : just add option without value
|
|
136
148
|
add_param=false
|
|
137
149
|
case parameter_value
|
|
138
|
-
when false# nothing to put on command line, no creation by default
|
|
139
|
-
when true
|
|
140
|
-
else raise Fasp::Error
|
|
150
|
+
when false then nil # nothing to put on command line, no creation by default
|
|
151
|
+
when true then add_param=true
|
|
152
|
+
else raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}"
|
|
141
153
|
end
|
|
142
|
-
add_param
|
|
143
|
-
add_command_line_options([options[:
|
|
154
|
+
add_param= !add_param if options[:add_on_false]
|
|
155
|
+
add_command_line_options([options[:clswitch]]) if add_param
|
|
144
156
|
when :opt_with_arg # transform into command line option with value
|
|
145
157
|
#parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
|
|
146
158
|
parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
|
|
147
159
|
# if transfer_spec value is an array, applies option many times
|
|
148
|
-
parameter_value.each{|v|add_command_line_options([options[:
|
|
160
|
+
parameter_value.each{|v|add_command_line_options([options[:clswitch],v])}
|
|
161
|
+
when NilClass
|
|
162
|
+
Log.log.debug("Ignoring parameter: #{param_name}")
|
|
149
163
|
else
|
|
150
|
-
raise "
|
|
164
|
+
raise "ERROR: unknown action: #{action}/#{action.class}"
|
|
151
165
|
end
|
|
152
166
|
end
|
|
153
167
|
end
|
data/lib/aspera/cos_node.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/log'
|
|
2
3
|
require 'aspera/rest'
|
|
3
4
|
require 'xmlsimple'
|
|
@@ -11,25 +12,28 @@ module Aspera
|
|
|
11
12
|
@auth_url=auth_url
|
|
12
13
|
@api_key=api_key
|
|
13
14
|
s3_api=Aspera::Rest.new({
|
|
14
|
-
:
|
|
15
|
-
:
|
|
16
|
-
:
|
|
17
|
-
:
|
|
18
|
-
|
|
19
|
-
:
|
|
20
|
-
:
|
|
21
|
-
:
|
|
22
|
-
|
|
15
|
+
base_url: storage_endpoint,
|
|
16
|
+
not_auth_codes: ['401','403'], # error codes when not authorized
|
|
17
|
+
headers: {'ibm-service-instance-id' => instance_id},
|
|
18
|
+
auth: {
|
|
19
|
+
type: :oauth2,
|
|
20
|
+
base_url: @auth_url,
|
|
21
|
+
crtype: :generic,
|
|
22
|
+
generic: {
|
|
23
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
|
24
|
+
response_type: 'cloud_iam',
|
|
25
|
+
apikey: @api_key
|
|
26
|
+
}}})
|
|
23
27
|
# read FASP connection information for bucket
|
|
24
|
-
xml_result_text=s3_api.call({:
|
|
28
|
+
xml_result_text=s3_api.call({operation: 'GET',subpath: bucket_name,headers: {'Accept'=>'application/xml'},url_params: {'faspConnectionInfo'=>nil}})[:http].body
|
|
25
29
|
ats_info=XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
|
|
26
30
|
Aspera::Log.dump('ats_info',ats_info)
|
|
27
31
|
super({
|
|
28
|
-
:
|
|
29
|
-
:
|
|
30
|
-
|
|
31
|
-
:
|
|
32
|
-
:
|
|
32
|
+
base_url: ats_info['ATSEndpoint'],
|
|
33
|
+
auth: {
|
|
34
|
+
type: :basic,
|
|
35
|
+
username: ats_info['AccessKey']['Id'],
|
|
36
|
+
password: ats_info['AccessKey']['Secret']}})
|
|
33
37
|
# prepare transfer spec addition
|
|
34
38
|
@add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
|
|
35
39
|
'type' => 'token',
|
|
@@ -42,12 +46,16 @@ module Aspera
|
|
|
42
46
|
def generate_token
|
|
43
47
|
# OAuth API to get delegated token
|
|
44
48
|
delegated_oauth=Oauth.new({
|
|
45
|
-
|
|
46
|
-
:
|
|
47
|
-
:
|
|
48
|
-
:
|
|
49
|
-
:
|
|
50
|
-
|
|
49
|
+
type: :oauth2,
|
|
50
|
+
base_url: @auth_url,
|
|
51
|
+
token_field: TOKEN_FIELD,
|
|
52
|
+
crtype: :generic,
|
|
53
|
+
generic: {
|
|
54
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
|
55
|
+
response_type: 'delegated_refresh_token',
|
|
56
|
+
apikey: @api_key,
|
|
57
|
+
receiver_client_ids: 'aspera_ats'
|
|
58
|
+
}})
|
|
51
59
|
# get delagated token to be placed in rest call header and in transfer tags
|
|
52
60
|
@add_ts['tags']['aspera']['node']['storage_credentials']['token'][TOKEN_FIELD]=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
|
|
53
61
|
@params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/log'
|
|
2
3
|
require 'singleton'
|
|
3
4
|
|
|
@@ -6,8 +7,8 @@ module Aspera
|
|
|
6
7
|
class DataRepository
|
|
7
8
|
include Singleton
|
|
8
9
|
# get binary value from data repository
|
|
9
|
-
def
|
|
10
|
-
File.read(File.join(
|
|
10
|
+
def data(id)
|
|
11
|
+
File.read(File.join(__dir__,'data',id.to_s),mode: 'rb')
|
|
11
12
|
end
|
|
12
13
|
end
|
|
13
14
|
end
|
data/lib/aspera/environment.rb
CHANGED
|
@@ -1,66 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/log'
|
|
2
3
|
require 'rbconfig'
|
|
3
4
|
|
|
4
5
|
module Aspera
|
|
5
|
-
# detect OS, architecture, and
|
|
6
|
+
# detect OS, architecture, and specific stuff
|
|
6
7
|
class Environment
|
|
7
8
|
OS_WINDOWS = :windows
|
|
8
9
|
OS_X = :osx
|
|
9
10
|
OS_LINUX = :linux
|
|
10
11
|
OS_AIX = :aix
|
|
11
|
-
OS_LIST=[OS_WINDOWS,OS_X,OS_LINUX,OS_AIX]
|
|
12
|
-
|
|
13
|
-
def self.os
|
|
14
|
-
case RbConfig::CONFIG['host_os']
|
|
15
|
-
when /mswin/,/msys/,/mingw/,/cygwin/,/bccwin/,/wince/,/emc/
|
|
16
|
-
return OS_WINDOWS
|
|
17
|
-
when /darwin/,/mac os/
|
|
18
|
-
return OS_X
|
|
19
|
-
when /linux/
|
|
20
|
-
return OS_LINUX
|
|
21
|
-
when /aix/
|
|
22
|
-
return OS_AIX
|
|
23
|
-
else
|
|
24
|
-
raise "Unknown OS: #{RbConfig::CONFIG['host_os']}"
|
|
25
|
-
end
|
|
26
|
-
end
|
|
27
|
-
CPU_X86_64=:x86_64
|
|
12
|
+
OS_LIST=[OS_WINDOWS,OS_X,OS_LINUX,OS_AIX].freeze
|
|
13
|
+
CPU_X86_64=:x86_64 # rubocop:disable Naming/VariableNumber
|
|
28
14
|
CPU_PPC64=:ppc64
|
|
29
15
|
CPU_PPC64LE=:ppc64le
|
|
30
16
|
CPU_S390=:s390
|
|
31
|
-
CPU_LIST=[CPU_X86_64,CPU_PPC64,CPU_PPC64LE,CPU_S390]
|
|
17
|
+
CPU_LIST=[CPU_X86_64,CPU_PPC64,CPU_PPC64LE,CPU_S390].freeze
|
|
32
18
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
return CPU_X86_64
|
|
37
|
-
when /powerpc/,/ppc64/
|
|
38
|
-
return CPU_PPC64LE if os.eql?(OS_LINUX)
|
|
39
|
-
return CPU_PPC64
|
|
40
|
-
when /s390/
|
|
41
|
-
return CPU_S390
|
|
42
|
-
else # other
|
|
43
|
-
raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
|
|
19
|
+
class<<self
|
|
20
|
+
def ruby_version
|
|
21
|
+
return RbConfig::CONFIG['RUBY_PROGRAM_VERSION']
|
|
44
22
|
end
|
|
45
|
-
end
|
|
46
23
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
24
|
+
def os
|
|
25
|
+
case RbConfig::CONFIG['host_os']
|
|
26
|
+
when /mswin/,/msys/,/mingw/,/cygwin/,/bccwin/,/wince/,/emc/
|
|
27
|
+
return OS_WINDOWS
|
|
28
|
+
when /darwin/,/mac os/
|
|
29
|
+
return OS_X
|
|
30
|
+
when /linux/
|
|
31
|
+
return OS_LINUX
|
|
32
|
+
when /aix/
|
|
33
|
+
return OS_AIX
|
|
34
|
+
else
|
|
35
|
+
raise "Unknown OS: #{RbConfig::CONFIG['host_os']}"
|
|
36
|
+
end
|
|
37
|
+
end
|
|
55
38
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
39
|
+
def cpu
|
|
40
|
+
case RbConfig::CONFIG['host_cpu']
|
|
41
|
+
when /x86_64/,/x64/
|
|
42
|
+
return CPU_X86_64
|
|
43
|
+
when /powerpc/,/ppc64/
|
|
44
|
+
return CPU_PPC64LE if os.eql?(OS_LINUX)
|
|
45
|
+
return CPU_PPC64
|
|
46
|
+
when /s390/
|
|
47
|
+
return CPU_S390
|
|
48
|
+
else # other
|
|
49
|
+
raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
|
|
62
50
|
end
|
|
63
51
|
end
|
|
52
|
+
|
|
53
|
+
def architecture
|
|
54
|
+
return "#{os}-#{cpu}"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def exe_extension
|
|
58
|
+
return '.exe' if os.eql?(OS_WINDOWS)
|
|
59
|
+
return ''
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# on Windows, the env var %USERPROFILE% provides the path to user's home more reliably than %HOMEDRIVE%%HOMEPATH%
|
|
63
|
+
# so, tell Ruby the right way
|
|
64
|
+
def fix_home
|
|
65
|
+
return unless os.eql?(OS_WINDOWS) && ENV.has_key?('USERPROFILE') && Dir.exist?(ENV['USERPROFILE'])
|
|
66
|
+
ENV['HOME']=ENV['USERPROFILE']
|
|
67
|
+
Log.log.debug("Windows: set home to USERPROFILE: #{ENV['HOME']}")
|
|
68
|
+
end
|
|
64
69
|
end
|
|
65
70
|
end
|
|
66
71
|
end
|