aspera-cli 4.4.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2095 -1503
- data/bin/ascli +2 -1
- data/bin/asession +4 -5
- data/docs/test_env.conf +3 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +25 -25
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +17 -17
- data/lib/aspera/aoc.rb +238 -185
- data/lib/aspera/ascmd.rb +93 -83
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +142 -108
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +18 -21
- data/lib/aspera/cli/main.rb +173 -149
- data/lib/aspera/cli/manager.rb +163 -168
- data/lib/aspera/cli/plugin.rb +43 -31
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +405 -370
- data/lib/aspera/cli/plugins/ats.rb +86 -79
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +580 -362
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +201 -158
- data/lib/aspera/cli/plugins/faspex5.rb +80 -57
- data/lib/aspera/cli/plugins/node.rb +183 -166
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +35 -19
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +76 -113
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
- data/lib/aspera/fasp/agent_trsdk.rb +104 -0
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +152 -124
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +87 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -89
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +121 -0
- data/lib/aspera/keychain/macos_security.rb +90 -0
- data/lib/aspera/log.rb +55 -37
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -25
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +154 -135
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +116 -29
- data/docs/Makefile +0 -66
- data/docs/README.erb.md +0 -3973
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/secrets.rb +0 -20
@@ -1,8 +1,5 @@
|
|
1
|
-
|
2
|
-
require 'aspera/fasp/
|
3
|
-
require 'aspera/fasp/connect'
|
4
|
-
require 'aspera/fasp/node'
|
5
|
-
require 'aspera/fasp/http_gw'
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'aspera/fasp/transfer_spec'
|
6
3
|
require 'aspera/cli/listener/logger'
|
7
4
|
require 'aspera/cli/listener/progress_multi'
|
8
5
|
|
@@ -16,21 +13,23 @@ module Aspera
|
|
16
13
|
FILE_LIST_FROM_ARGS='@args'
|
17
14
|
# special value for --sources : read file list from transfer spec (--ts)
|
18
15
|
FILE_LIST_FROM_TRANSFER_SPEC='@ts'
|
19
|
-
DEFAULT_TRANSFER_NOTIF_TMPL
|
20
|
-
From: <%=from_name%> <<%=from_email%>>
|
21
|
-
To: <<%=to%>>
|
22
|
-
Subject: <%=subject%>
|
16
|
+
DEFAULT_TRANSFER_NOTIF_TMPL=<<~END_OF_TEMPLATE
|
17
|
+
From: <%=from_name%> <<%=from_email%>>
|
18
|
+
To: <<%=to%>>
|
19
|
+
Subject: <%=subject%>
|
23
20
|
|
24
|
-
Transfer is: <%=global_transfer_status%>
|
21
|
+
Transfer is: <%=global_transfer_status%>
|
25
22
|
|
26
|
-
<%=ts.to_yaml%>
|
27
|
-
END_OF_TEMPLATE
|
28
|
-
#%
|
23
|
+
<%=ts.to_yaml%>
|
24
|
+
END_OF_TEMPLATE
|
25
|
+
#% (formating bug in eclipse)
|
29
26
|
private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
|
27
|
+
TRANSFER_AGENTS=[:direct,:node,:connect,:httpgw,:trsdk]
|
28
|
+
|
30
29
|
# @param env external objects: option manager, config file manager
|
31
|
-
def initialize(
|
32
|
-
|
33
|
-
@
|
30
|
+
def initialize(opt_mgr,config)
|
31
|
+
@opt_mgr=opt_mgr
|
32
|
+
@config=config
|
34
33
|
# command line can override transfer spec
|
35
34
|
@transfer_spec_cmdline={'create_dir'=>true}
|
36
35
|
# the currently selected transfer agent
|
@@ -38,25 +37,21 @@ END_OF_TEMPLATE
|
|
38
37
|
@progress_listener=Listener::ProgressMulti.new
|
39
38
|
# source/destination pair, like "paths" of transfer spec
|
40
39
|
@transfer_paths=nil
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
40
|
+
@opt_mgr.set_obj_attr(:ts,self,:option_transfer_spec)
|
41
|
+
@opt_mgr.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:ts,:optional)}")
|
42
|
+
@opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
|
43
|
+
@opt_mgr.add_opt_simple(:to_folder,'destination folder for downloaded files')
|
44
|
+
@opt_mgr.add_opt_simple(:sources,'list of source files (see doc)')
|
45
|
+
@opt_mgr.add_opt_simple(:transfer_info,'parameters for transfer agent')
|
46
|
+
@opt_mgr.add_opt_list(:src_type,[:list,:pair],'type of file list')
|
47
|
+
@opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,'type of transfer agent')
|
48
|
+
@opt_mgr.add_opt_list(:progress,[:none,:native,:multi],'type of progress bar')
|
49
|
+
@opt_mgr.set_option(:transfer,:direct)
|
50
|
+
@opt_mgr.set_option(:src_type,:list)
|
51
|
+
@opt_mgr.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
|
52
|
+
@opt_mgr.parse_options!
|
54
53
|
end
|
55
54
|
|
56
|
-
def options; @env[:options];end
|
57
|
-
|
58
|
-
def config; @env[:config];end
|
59
|
-
|
60
55
|
def option_transfer_spec; @transfer_spec_cmdline; end
|
61
56
|
|
62
57
|
# multiple option are merged
|
@@ -64,12 +59,12 @@ END_OF_TEMPLATE
|
|
64
59
|
|
65
60
|
def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
|
66
61
|
|
67
|
-
def
|
62
|
+
def agent_instance=(instance)
|
68
63
|
@agent=instance
|
69
64
|
@agent.add_listener(Listener::Logger.new)
|
70
65
|
# use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
|
71
|
-
if
|
72
|
-
(
|
66
|
+
if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) ||
|
67
|
+
(@opt_mgr.get_option(:progress,:mandatory).eql?(:native) && !instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
|
73
68
|
@agent.add_listener(@progress_listener)
|
74
69
|
end
|
75
70
|
end
|
@@ -77,57 +72,25 @@ END_OF_TEMPLATE
|
|
77
72
|
# analyze options and create new agent if not already created or set
|
78
73
|
def set_agent_by_options
|
79
74
|
return nil unless @agent.nil?
|
80
|
-
agent_type
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
# support: @preset:<name>
|
95
|
-
# support extended values
|
96
|
-
node_config=options.get_option(:transfer_info,:optional)
|
97
|
-
# if not specified: use default node
|
98
|
-
if node_config.nil?
|
99
|
-
param_set_name=config.get_plugin_default_config_name(:node)
|
100
|
-
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
101
|
-
node_config=config.preset_by_name(param_set_name)
|
102
|
-
end
|
103
|
-
Log.log.debug("node=#{node_config}")
|
104
|
-
raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" unless node_config.is_a?(Hash)
|
105
|
-
# here, node_config is a Hash
|
106
|
-
node_config=node_config.symbolize_keys
|
107
|
-
# Check mandatory params
|
108
|
-
[:url,:username,:password].each { |k| raise CliBadArgument,"missing parameter [#{k}] in node specification: #{node_config}" unless node_config.has_key?(k) }
|
109
|
-
if node_config[:password].match(/^Bearer /)
|
110
|
-
node_api=Rest.new({
|
111
|
-
base_url: node_config[:url],
|
112
|
-
headers: {
|
113
|
-
'X-Aspera-AccessKey'=>node_config[:username],
|
114
|
-
'Authorization' =>node_config[:password]}})
|
115
|
-
else
|
116
|
-
node_api=Rest.new({
|
117
|
-
base_url: node_config[:url],
|
118
|
-
auth: {
|
119
|
-
type: :basic,
|
120
|
-
username: node_config[:username],
|
121
|
-
password: node_config[:password]
|
122
|
-
}})
|
123
|
-
end
|
124
|
-
new_agent=Fasp::Node.new(node_api)
|
125
|
-
# add root id if it's an access key
|
126
|
-
new_agent.options={root_id: node_config[:root_id]} if node_config.has_key?(:root_id)
|
127
|
-
else
|
128
|
-
raise "Unexpected transfer agent type: #{agent_type}"
|
75
|
+
agent_type=@opt_mgr.get_option(:transfer,:mandatory)
|
76
|
+
require "aspera/fasp/agent_#{agent_type}"
|
77
|
+
agent_options=@opt_mgr.get_option(:transfer_info,:optional)
|
78
|
+
raise CliBadArgument,"the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), use either @json:<json> or @preset:<parameter set name>" unless [Hash,NilClass].include?(agent_options.class)
|
79
|
+
# special case
|
80
|
+
if agent_type.eql?(:node) && agent_options.nil?
|
81
|
+
param_set_name=@config.get_plugin_default_config_name(:node)
|
82
|
+
raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
|
83
|
+
agent_options=@config.preset_by_name(param_set_name)
|
84
|
+
end
|
85
|
+
# special case
|
86
|
+
if agent_type.eql?(:direct) && @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
|
87
|
+
agent_options={} if agent_options.nil?
|
88
|
+
agent_options[:quiet]=false
|
129
89
|
end
|
130
|
-
|
90
|
+
agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
|
91
|
+
# get agent instance
|
92
|
+
new_agent=Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
|
93
|
+
self.agent_instance=new_agent
|
131
94
|
return nil
|
132
95
|
end
|
133
96
|
|
@@ -135,14 +98,14 @@ END_OF_TEMPLATE
|
|
135
98
|
# sets default if needed
|
136
99
|
# param: 'send' or 'receive'
|
137
100
|
def destination_folder(direction)
|
138
|
-
dest_folder
|
139
|
-
return dest_folder unless dest_folder.nil?
|
101
|
+
dest_folder=@opt_mgr.get_option(:to_folder,:optional)
|
102
|
+
return File.expand_path(dest_folder) unless dest_folder.nil?
|
140
103
|
dest_folder=@transfer_spec_cmdline['destination_root']
|
141
104
|
return dest_folder unless dest_folder.nil?
|
142
105
|
# default: / on remote, . on local
|
143
106
|
case direction.to_s
|
144
|
-
when
|
145
|
-
when
|
107
|
+
when Fasp::TransferSpec::DIRECTION_SEND then dest_folder='/'
|
108
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE then dest_folder='.'
|
146
109
|
else raise "wrong direction: #{direction}"
|
147
110
|
end
|
148
111
|
return dest_folder
|
@@ -150,7 +113,7 @@ END_OF_TEMPLATE
|
|
150
113
|
|
151
114
|
# This is how the list of files to be transfered is specified
|
152
115
|
# get paths suitable for transfer spec from command line
|
153
|
-
# @return {:
|
116
|
+
# @return {source: (mandatory), destination: (optional)}
|
154
117
|
# computation is done only once, cache is kept in @transfer_paths
|
155
118
|
def ts_source_paths
|
156
119
|
# return cache if set
|
@@ -158,37 +121,37 @@ END_OF_TEMPLATE
|
|
158
121
|
# start with lower priority : get paths from transfer spec on command line
|
159
122
|
@transfer_paths=@transfer_spec_cmdline['paths'] if @transfer_spec_cmdline.has_key?('paths')
|
160
123
|
# is there a source list option ?
|
161
|
-
file_list
|
124
|
+
file_list=@opt_mgr.get_option(:sources,:optional)
|
162
125
|
case file_list
|
163
126
|
when nil,FILE_LIST_FROM_ARGS
|
164
|
-
Log.log.debug(
|
127
|
+
Log.log.debug('getting file list as parameters')
|
165
128
|
# get remaining arguments
|
166
|
-
file_list
|
167
|
-
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array)
|
129
|
+
file_list=@opt_mgr.get_next_argument('source file list',:multiple)
|
130
|
+
raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) || file_list.empty?
|
168
131
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
169
|
-
Log.log.debug(
|
170
|
-
special_case_direct_with_list
|
171
|
-
raise CliBadArgument,
|
132
|
+
Log.log.debug('assume list provided in transfer spec')
|
133
|
+
special_case_direct_with_list=@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
|
134
|
+
raise CliBadArgument,'transfer spec on command line must have sources' if @transfer_paths.nil? && !special_case_direct_with_list
|
172
135
|
# here we assume check of sources is made in transfer agent
|
173
136
|
return @transfer_paths
|
174
137
|
when Array
|
175
|
-
Log.log.debug(
|
176
|
-
raise CliBadArgument,
|
138
|
+
Log.log.debug('getting file list as extended value')
|
139
|
+
raise CliBadArgument,'sources must be a Array of String' if !file_list.reject{|f|f.is_a?(String)}.empty?
|
177
140
|
else
|
178
141
|
raise CliBadArgument,"sources must be a Array, not #{file_list.class}"
|
179
142
|
end
|
180
143
|
# here, file_list is an Array or String
|
181
144
|
if !@transfer_paths.nil?
|
182
|
-
Log.log.warn(
|
145
|
+
Log.log.warn('--sources overrides paths from --ts')
|
183
146
|
end
|
184
|
-
case
|
147
|
+
case @opt_mgr.get_option(:src_type,:mandatory)
|
185
148
|
when :list
|
186
149
|
# when providing a list, just specify source
|
187
150
|
@transfer_paths=file_list.map{|i|{'source'=>i}}
|
188
151
|
when :pair
|
189
152
|
raise CliBadArgument,"When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
|
190
153
|
@transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
|
191
|
-
else raise
|
154
|
+
else raise 'Unsupported src_type'
|
192
155
|
end
|
193
156
|
Log.log.debug("paths=#{@transfer_paths}")
|
194
157
|
return @transfer_paths
|
@@ -201,14 +164,14 @@ END_OF_TEMPLATE
|
|
201
164
|
# other options are carried to specific agent
|
202
165
|
def start(transfer_spec,tr_opts)
|
203
166
|
# check parameters
|
204
|
-
raise
|
205
|
-
raise
|
167
|
+
raise 'transfer_spec must be hash' unless transfer_spec.is_a?(Hash)
|
168
|
+
raise 'tr_opts must be hash' unless tr_opts.is_a?(Hash)
|
206
169
|
# process :src option
|
207
170
|
case transfer_spec['direction']
|
208
|
-
when
|
171
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE
|
209
172
|
# init default if required in any case
|
210
173
|
@transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
|
211
|
-
when
|
174
|
+
when Fasp::TransferSpec::DIRECTION_SEND
|
212
175
|
case tr_opts[:src]
|
213
176
|
when :direct
|
214
177
|
# init default if required
|
@@ -232,18 +195,19 @@ END_OF_TEMPLATE
|
|
232
195
|
|
233
196
|
transfer_spec.merge!(@transfer_spec_cmdline)
|
234
197
|
# create transfer agent
|
235
|
-
|
198
|
+
set_agent_by_options
|
236
199
|
Log.log.debug("transfer agent is a #{@agent.class}")
|
237
200
|
@agent.start_transfer(transfer_spec,tr_opts)
|
201
|
+
# list of : :success or error message
|
238
202
|
result=@agent.wait_for_transfers_completion
|
239
203
|
@progress_listener.reset
|
240
|
-
Fasp::
|
204
|
+
Fasp::AgentBase.validate_status_list(result)
|
241
205
|
send_email_transfer_notification(transfer_spec,result)
|
242
206
|
return result
|
243
207
|
end
|
244
208
|
|
245
209
|
def send_email_transfer_notification(transfer_spec,statuses)
|
246
|
-
return if
|
210
|
+
return if @opt_mgr.get_option(:notif_to,:optional).nil?
|
247
211
|
global_status=self.class.session_status(statuses)
|
248
212
|
email_vars={
|
249
213
|
global_transfer_status: global_status,
|
@@ -251,13 +215,13 @@ END_OF_TEMPLATE
|
|
251
215
|
body: "Transfer is: #{global_status}",
|
252
216
|
ts: transfer_spec
|
253
217
|
}
|
254
|
-
@
|
218
|
+
@config.send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
|
255
219
|
end
|
256
220
|
|
257
221
|
# @return :success if all sessions statuses returned by "start" are success
|
258
222
|
# else return the first error exception object
|
259
223
|
def self.session_status(statuses)
|
260
|
-
error_statuses=statuses.
|
224
|
+
error_statuses=statuses.reject{|i|i.eql?(:success)}
|
261
225
|
return :success if error_statuses.empty?
|
262
226
|
return error_statuses.first
|
263
227
|
end
|
@@ -266,7 +230,6 @@ END_OF_TEMPLATE
|
|
266
230
|
def shutdown
|
267
231
|
@agent.shutdown if @agent.respond_to?(:shutdown)
|
268
232
|
end
|
269
|
-
|
270
233
|
end
|
271
234
|
end
|
272
235
|
end
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/colors.rb
CHANGED
@@ -1,43 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
# simple vt100 colors
|
2
3
|
class String
|
3
|
-
|
4
|
-
|
4
|
+
class<<self
|
5
|
+
private
|
6
|
+
|
7
|
+
def vtcmd(code);"\e[#{code}m";end
|
8
|
+
end
|
5
9
|
# see https://en.wikipedia.org/wiki/ANSI_escape_code
|
6
10
|
# symbol is the method name added to String
|
7
11
|
# it adds control chars to set color (and reset at the end).
|
8
12
|
VTSTYLES = {
|
9
|
-
:
|
10
|
-
:
|
11
|
-
:
|
12
|
-
:
|
13
|
-
:
|
14
|
-
:
|
15
|
-
:
|
16
|
-
:
|
17
|
-
:
|
18
|
-
:
|
19
|
-
:
|
20
|
-
:
|
21
|
-
:
|
22
|
-
:
|
23
|
-
:
|
24
|
-
:
|
25
|
-
:
|
26
|
-
:
|
27
|
-
:
|
28
|
-
:
|
29
|
-
:
|
13
|
+
bold: 1,
|
14
|
+
italic: 3,
|
15
|
+
underline: 4,
|
16
|
+
blink: 5,
|
17
|
+
reverse_color: 7,
|
18
|
+
black: 30,
|
19
|
+
red: 31,
|
20
|
+
green: 32,
|
21
|
+
brown: 33,
|
22
|
+
blue: 34,
|
23
|
+
magenta: 35,
|
24
|
+
cyan: 36,
|
25
|
+
gray: 37,
|
26
|
+
bg_black: 40,
|
27
|
+
bg_red: 41,
|
28
|
+
bg_green: 42,
|
29
|
+
bg_brown: 43,
|
30
|
+
bg_blue: 44,
|
31
|
+
bg_magenta: 45,
|
32
|
+
bg_cyan: 46,
|
33
|
+
bg_gray: 47
|
30
34
|
}
|
31
35
|
private_constant :VTSTYLES
|
32
36
|
# defines methods to String, one per entry in VTSTYLES
|
33
37
|
VTSTYLES.each do |name,code|
|
34
|
-
|
35
|
-
|
36
|
-
|
38
|
+
if $stderr.tty?
|
39
|
+
begin_seq=vtcmd(code)
|
40
|
+
end_code=
|
41
|
+
if code >= 10 then 0
|
42
|
+
elsif code.eql?(1) then 22
|
43
|
+
else code+20
|
44
|
+
end
|
45
|
+
end_seq=vtcmd(end_code)
|
37
46
|
define_method(name){"#{begin_seq}#{self}#{end_seq}"}
|
38
47
|
else
|
39
48
|
define_method(name){self}
|
40
49
|
end
|
41
|
-
public name
|
42
50
|
end
|
43
51
|
end
|
@@ -1,14 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module Aspera
|
2
3
|
# helper class to build command line from a parameter list (key-value hash)
|
3
4
|
# constructor takes hash: { 'param1':'value1', ...}
|
4
5
|
# process_param is called repeatedly with all known parameters
|
5
6
|
# add_env_args is called to get resulting param list and env var (also checks that all params were used)
|
6
7
|
class CommandLineBuilder
|
7
|
-
# transform yes/no to
|
8
|
+
# transform yes/no to true/false
|
8
9
|
def self.yes_to_true(value)
|
9
10
|
case value
|
10
|
-
when 'yes'
|
11
|
-
when 'no'
|
11
|
+
when 'yes' then return true
|
12
|
+
when 'no' then return false
|
12
13
|
end
|
13
14
|
raise "unsupported value: #{value}"
|
14
15
|
end
|
@@ -22,13 +23,13 @@ module Aspera
|
|
22
23
|
options[:mandatory]||=false
|
23
24
|
options[:desc]||=''
|
24
25
|
# by default : string, unless it's without arg
|
25
|
-
if !
|
26
|
+
if !options.has_key?(:accepted_types)
|
26
27
|
options[:accepted_types]=options[:cltype].eql?(:opt_without_arg) ? :bool : :string
|
27
28
|
end
|
28
29
|
# single type is placed in array
|
29
30
|
options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
|
30
|
-
if !options.has_key?(:
|
31
|
-
options[:
|
31
|
+
if !options.has_key?(:clswitch) && options.has_key?(:cltype) && [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
|
32
|
+
options[:clswitch]='--'+param_name.to_s.gsub('_','-')
|
32
33
|
end
|
33
34
|
end
|
34
35
|
end
|
@@ -36,7 +37,7 @@ module Aspera
|
|
36
37
|
private
|
37
38
|
|
38
39
|
# clvarname : command line variable name
|
39
|
-
def env_name(
|
40
|
+
def env_name(_param_name,options)
|
40
41
|
return options[:clvarname]
|
41
42
|
end
|
42
43
|
|
@@ -46,7 +47,7 @@ module Aspera
|
|
46
47
|
|
47
48
|
# @param param_hash
|
48
49
|
def initialize(param_hash,params_definition)
|
49
|
-
@param_hash=param_hash
|
50
|
+
@param_hash=param_hash # keep reference so that it can be modified by caller before calling `process_params`
|
50
51
|
@params_definition=params_definition
|
51
52
|
@result_env={}
|
52
53
|
@result_args=[]
|
@@ -86,42 +87,53 @@ module Aspera
|
|
86
87
|
# @param options : options for type
|
87
88
|
def process_param(param_name,action=nil)
|
88
89
|
options=@params_definition[param_name]
|
89
|
-
action=options[:cltype] if action.nil?
|
90
90
|
# should not happen
|
91
|
-
|
91
|
+
if options.nil?
|
92
|
+
Log.log.warn("Unknown parameter #{param_name}")
|
93
|
+
return
|
94
|
+
end
|
95
|
+
action=options[:cltype] if action.nil?
|
92
96
|
# check mandatory parameter (nil is valid value)
|
93
|
-
raise Fasp::Error
|
97
|
+
raise Fasp::Error, "Missing mandatory parameter: #{param_name}" if options[:mandatory] && !@param_hash.has_key?(param_name)
|
94
98
|
parameter_value=@param_hash[param_name]
|
99
|
+
|
95
100
|
#parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
|
101
|
+
|
102
|
+
# Check parameter type
|
96
103
|
expected_classes=options[:accepted_types].map do |s|
|
97
104
|
case s
|
98
|
-
when :string
|
99
|
-
when :array
|
100
|
-
when :hash
|
101
|
-
when :int
|
102
|
-
when :bool
|
105
|
+
when :string then String
|
106
|
+
when :array then Array
|
107
|
+
when :hash then Hash
|
108
|
+
when :int then Integer
|
109
|
+
when :bool then [TrueClass,FalseClass]
|
103
110
|
else raise "INTERNAL: unexpected value: #{s}"
|
104
111
|
end
|
105
112
|
end.flatten
|
106
|
-
#
|
107
|
-
raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or expected_classes.include?(parameter_value.class)
|
113
|
+
raise Fasp::Error,"#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, " unless parameter_value.nil? || expected_classes.include?(parameter_value.class)
|
108
114
|
@used_param_names.push(param_name) unless action.eql?(:defer)
|
109
115
|
|
110
116
|
# process only non-nil values
|
111
117
|
return nil if parameter_value.nil?
|
112
118
|
|
113
|
-
|
119
|
+
# check that value is of an accepted type (string, int bool)
|
120
|
+
raise "Value #{parameter_value} is not allowed for #{param_name}" if options.has_key?(:enum) && !options[:enum].include?(parameter_value)
|
121
|
+
|
122
|
+
# convert some values if value on command line needs processing from value in structure
|
123
|
+
case options[:clconvert]
|
124
|
+
when Hash
|
114
125
|
# translate using conversion table
|
115
|
-
new_value=options[:
|
116
|
-
raise "unsupported value: #{parameter_value}" if new_value.nil?
|
126
|
+
new_value=options[:clconvert][parameter_value]
|
127
|
+
raise "unsupported value: #{parameter_value}, expect: #{options[:clconvert].keys.join(', ')}" if new_value.nil?
|
117
128
|
parameter_value=new_value
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
|
129
|
+
when String
|
130
|
+
# :clconvert has name of class and encoding method
|
131
|
+
convclass,convmethod=options[:clconvert].split('.')
|
132
|
+
newvalue=Kernel.const_get(convclass).send(convmethod,parameter_value)
|
133
|
+
raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}" if newvalue.nil?
|
124
134
|
parameter_value=newvalue
|
135
|
+
when NilClass
|
136
|
+
else raise "not expected type for clconvert #{options[:clconvert].class} for #{param_name}"
|
125
137
|
end
|
126
138
|
|
127
139
|
case action
|
@@ -135,19 +147,21 @@ module Aspera
|
|
135
147
|
when :opt_without_arg # if present and true : just add option without value
|
136
148
|
add_param=false
|
137
149
|
case parameter_value
|
138
|
-
when false# nothing to put on command line, no creation by default
|
139
|
-
when true
|
140
|
-
else raise Fasp::Error
|
150
|
+
when false then nil # nothing to put on command line, no creation by default
|
151
|
+
when true then add_param=true
|
152
|
+
else raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}"
|
141
153
|
end
|
142
|
-
add_param
|
143
|
-
add_command_line_options([options[:
|
154
|
+
add_param= !add_param if options[:add_on_false]
|
155
|
+
add_command_line_options([options[:clswitch]]) if add_param
|
144
156
|
when :opt_with_arg # transform into command line option with value
|
145
157
|
#parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
|
146
158
|
parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
|
147
159
|
# if transfer_spec value is an array, applies option many times
|
148
|
-
parameter_value.each{|v|add_command_line_options([options[:
|
160
|
+
parameter_value.each{|v|add_command_line_options([options[:clswitch],v])}
|
161
|
+
when NilClass
|
162
|
+
Log.log.debug("Ignoring parameter: #{param_name}")
|
149
163
|
else
|
150
|
-
raise "
|
164
|
+
raise "ERROR: unknown action: #{action}/#{action.class}"
|
151
165
|
end
|
152
166
|
end
|
153
167
|
end
|
data/lib/aspera/cos_node.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/log'
|
2
3
|
require 'aspera/rest'
|
3
4
|
require 'xmlsimple'
|
@@ -11,25 +12,28 @@ module Aspera
|
|
11
12
|
@auth_url=auth_url
|
12
13
|
@api_key=api_key
|
13
14
|
s3_api=Aspera::Rest.new({
|
14
|
-
:
|
15
|
-
:
|
16
|
-
:
|
17
|
-
:
|
18
|
-
|
19
|
-
:
|
20
|
-
:
|
21
|
-
:
|
22
|
-
|
15
|
+
base_url: storage_endpoint,
|
16
|
+
not_auth_codes: ['401','403'], # error codes when not authorized
|
17
|
+
headers: {'ibm-service-instance-id' => instance_id},
|
18
|
+
auth: {
|
19
|
+
type: :oauth2,
|
20
|
+
base_url: @auth_url,
|
21
|
+
crtype: :generic,
|
22
|
+
generic: {
|
23
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
24
|
+
response_type: 'cloud_iam',
|
25
|
+
apikey: @api_key
|
26
|
+
}}})
|
23
27
|
# read FASP connection information for bucket
|
24
|
-
xml_result_text=s3_api.call({:
|
28
|
+
xml_result_text=s3_api.call({operation: 'GET',subpath: bucket_name,headers: {'Accept'=>'application/xml'},url_params: {'faspConnectionInfo'=>nil}})[:http].body
|
25
29
|
ats_info=XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
|
26
30
|
Aspera::Log.dump('ats_info',ats_info)
|
27
31
|
super({
|
28
|
-
:
|
29
|
-
:
|
30
|
-
|
31
|
-
:
|
32
|
-
:
|
32
|
+
base_url: ats_info['ATSEndpoint'],
|
33
|
+
auth: {
|
34
|
+
type: :basic,
|
35
|
+
username: ats_info['AccessKey']['Id'],
|
36
|
+
password: ats_info['AccessKey']['Secret']}})
|
33
37
|
# prepare transfer spec addition
|
34
38
|
@add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
|
35
39
|
'type' => 'token',
|
@@ -42,12 +46,16 @@ module Aspera
|
|
42
46
|
def generate_token
|
43
47
|
# OAuth API to get delegated token
|
44
48
|
delegated_oauth=Oauth.new({
|
45
|
-
|
46
|
-
:
|
47
|
-
:
|
48
|
-
:
|
49
|
-
:
|
50
|
-
|
49
|
+
type: :oauth2,
|
50
|
+
base_url: @auth_url,
|
51
|
+
token_field: TOKEN_FIELD,
|
52
|
+
crtype: :generic,
|
53
|
+
generic: {
|
54
|
+
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
|
55
|
+
response_type: 'delegated_refresh_token',
|
56
|
+
apikey: @api_key,
|
57
|
+
receiver_client_ids: 'aspera_ats'
|
58
|
+
}})
|
51
59
|
# get delagated token to be placed in rest call header and in transfer tags
|
52
60
|
@add_ts['tags']['aspera']['node']['storage_credentials']['token'][TOKEN_FIELD]=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
|
53
61
|
@params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/log'
|
2
3
|
require 'singleton'
|
3
4
|
|
@@ -6,8 +7,8 @@ module Aspera
|
|
6
7
|
class DataRepository
|
7
8
|
include Singleton
|
8
9
|
# get binary value from data repository
|
9
|
-
def
|
10
|
-
File.read(File.join(
|
10
|
+
def data(id)
|
11
|
+
File.read(File.join(__dir__,'data',id.to_s),mode: 'rb')
|
11
12
|
end
|
12
13
|
end
|
13
14
|
end
|