aspera-cli 4.4.0 → 4.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2095 -1503
  3. data/bin/ascli +2 -1
  4. data/bin/asession +4 -5
  5. data/docs/test_env.conf +3 -0
  6. data/examples/aoc.rb +4 -3
  7. data/examples/faspex4.rb +25 -25
  8. data/examples/proxy.pac +1 -1
  9. data/examples/transfer.rb +17 -17
  10. data/lib/aspera/aoc.rb +238 -185
  11. data/lib/aspera/ascmd.rb +93 -83
  12. data/lib/aspera/ats_api.rb +11 -10
  13. data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
  14. data/lib/aspera/cli/extended_value.rb +42 -33
  15. data/lib/aspera/cli/formater.rb +142 -108
  16. data/lib/aspera/cli/info.rb +17 -0
  17. data/lib/aspera/cli/listener/line_dump.rb +3 -2
  18. data/lib/aspera/cli/listener/logger.rb +2 -1
  19. data/lib/aspera/cli/listener/progress.rb +16 -18
  20. data/lib/aspera/cli/listener/progress_multi.rb +18 -21
  21. data/lib/aspera/cli/main.rb +173 -149
  22. data/lib/aspera/cli/manager.rb +163 -168
  23. data/lib/aspera/cli/plugin.rb +43 -31
  24. data/lib/aspera/cli/plugins/alee.rb +6 -6
  25. data/lib/aspera/cli/plugins/aoc.rb +405 -370
  26. data/lib/aspera/cli/plugins/ats.rb +86 -79
  27. data/lib/aspera/cli/plugins/bss.rb +14 -16
  28. data/lib/aspera/cli/plugins/config.rb +580 -362
  29. data/lib/aspera/cli/plugins/console.rb +23 -19
  30. data/lib/aspera/cli/plugins/cos.rb +18 -18
  31. data/lib/aspera/cli/plugins/faspex.rb +201 -158
  32. data/lib/aspera/cli/plugins/faspex5.rb +80 -57
  33. data/lib/aspera/cli/plugins/node.rb +183 -166
  34. data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
  35. data/lib/aspera/cli/plugins/preview.rb +92 -96
  36. data/lib/aspera/cli/plugins/server.rb +79 -75
  37. data/lib/aspera/cli/plugins/shares.rb +35 -19
  38. data/lib/aspera/cli/plugins/sync.rb +20 -22
  39. data/lib/aspera/cli/transfer_agent.rb +76 -113
  40. data/lib/aspera/cli/version.rb +2 -1
  41. data/lib/aspera/colors.rb +35 -27
  42. data/lib/aspera/command_line_builder.rb +48 -34
  43. data/lib/aspera/cos_node.rb +29 -21
  44. data/lib/aspera/data_repository.rb +3 -2
  45. data/lib/aspera/environment.rb +50 -45
  46. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
  47. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
  48. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
  49. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
  50. data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
  51. data/lib/aspera/fasp/agent_trsdk.rb +104 -0
  52. data/lib/aspera/fasp/error.rb +2 -1
  53. data/lib/aspera/fasp/error_info.rb +68 -52
  54. data/lib/aspera/fasp/installation.rb +152 -124
  55. data/lib/aspera/fasp/listener.rb +1 -0
  56. data/lib/aspera/fasp/parameters.rb +87 -92
  57. data/lib/aspera/fasp/parameters.yaml +305 -249
  58. data/lib/aspera/fasp/resume_policy.rb +11 -14
  59. data/lib/aspera/fasp/transfer_spec.rb +26 -0
  60. data/lib/aspera/fasp/uri.rb +22 -21
  61. data/lib/aspera/faspex_gw.rb +55 -89
  62. data/lib/aspera/hash_ext.rb +4 -3
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +121 -0
  65. data/lib/aspera/keychain/macos_security.rb +90 -0
  66. data/lib/aspera/log.rb +55 -37
  67. data/lib/aspera/nagios.rb +13 -12
  68. data/lib/aspera/node.rb +30 -25
  69. data/lib/aspera/oauth.rb +175 -226
  70. data/lib/aspera/open_application.rb +4 -3
  71. data/lib/aspera/persistency_action_once.rb +6 -6
  72. data/lib/aspera/persistency_folder.rb +5 -9
  73. data/lib/aspera/preview/file_types.rb +6 -5
  74. data/lib/aspera/preview/generator.rb +25 -24
  75. data/lib/aspera/preview/options.rb +16 -14
  76. data/lib/aspera/preview/utils.rb +98 -98
  77. data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
  78. data/lib/aspera/proxy_auto_config.rb +111 -20
  79. data/lib/aspera/rest.rb +154 -135
  80. data/lib/aspera/rest_call_error.rb +2 -2
  81. data/lib/aspera/rest_error_analyzer.rb +23 -25
  82. data/lib/aspera/rest_errors_aspera.rb +15 -14
  83. data/lib/aspera/ssh.rb +12 -10
  84. data/lib/aspera/sync.rb +42 -41
  85. data/lib/aspera/temp_file_manager.rb +18 -14
  86. data/lib/aspera/timer_limiter.rb +2 -1
  87. data/lib/aspera/uri_reader.rb +7 -5
  88. data/lib/aspera/web_auth.rb +79 -76
  89. metadata +116 -29
  90. data/docs/Makefile +0 -66
  91. data/docs/README.erb.md +0 -3973
  92. data/docs/README.md +0 -13
  93. data/docs/diagrams.txt +0 -49
  94. data/docs/doc_tools.rb +0 -58
  95. data/lib/aspera/api_detector.rb +0 -60
  96. data/lib/aspera/cli/plugins/shares2.rb +0 -114
  97. data/lib/aspera/secrets.rb +0 -20
@@ -1,8 +1,5 @@
1
- require 'aspera/fasp/local'
2
- require 'aspera/fasp/parameters'
3
- require 'aspera/fasp/connect'
4
- require 'aspera/fasp/node'
5
- require 'aspera/fasp/http_gw'
1
+ # frozen_string_literal: true
2
+ require 'aspera/fasp/transfer_spec'
6
3
  require 'aspera/cli/listener/logger'
7
4
  require 'aspera/cli/listener/progress_multi'
8
5
 
@@ -16,21 +13,23 @@ module Aspera
16
13
  FILE_LIST_FROM_ARGS='@args'
17
14
  # special value for --sources : read file list from transfer spec (--ts)
18
15
  FILE_LIST_FROM_TRANSFER_SPEC='@ts'
19
- DEFAULT_TRANSFER_NOTIF_TMPL=<<END_OF_TEMPLATE
20
- From: <%=from_name%> <<%=from_email%>>
21
- To: <<%=to%>>
22
- Subject: <%=subject%>
16
+ DEFAULT_TRANSFER_NOTIF_TMPL=<<~END_OF_TEMPLATE
17
+ From: <%=from_name%> <<%=from_email%>>
18
+ To: <<%=to%>>
19
+ Subject: <%=subject%>
23
20
 
24
- Transfer is: <%=global_transfer_status%>
21
+ Transfer is: <%=global_transfer_status%>
25
22
 
26
- <%=ts.to_yaml%>
27
- END_OF_TEMPLATE
28
- #%
23
+ <%=ts.to_yaml%>
24
+ END_OF_TEMPLATE
25
+ #% (formating bug in eclipse)
29
26
  private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:DEFAULT_TRANSFER_NOTIF_TMPL
27
+ TRANSFER_AGENTS=[:direct,:node,:connect,:httpgw,:trsdk]
28
+
30
29
  # @param env external objects: option manager, config file manager
31
- def initialize(env)
32
- # same as plugin environment
33
- @env=env
30
+ def initialize(opt_mgr,config)
31
+ @opt_mgr=opt_mgr
32
+ @config=config
34
33
  # command line can override transfer spec
35
34
  @transfer_spec_cmdline={'create_dir'=>true}
36
35
  # the currently selected transfer agent
@@ -38,25 +37,21 @@ END_OF_TEMPLATE
38
37
  @progress_listener=Listener::ProgressMulti.new
39
38
  # source/destination pair, like "paths" of transfer spec
40
39
  @transfer_paths=nil
41
- options.set_obj_attr(:ts,self,:option_transfer_spec)
42
- options.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{options.get_option(:ts,:optional)}")
43
- options.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{options.get_option(:local_resume,:optional)}")
44
- options.add_opt_simple(:to_folder,"destination folder for downloaded files")
45
- options.add_opt_simple(:sources,"list of source files (see doc)")
46
- options.add_opt_simple(:transfer_info,"parameters for transfer agent")
47
- options.add_opt_list(:src_type,[:list,:pair],"type of file list")
48
- options.add_opt_list(:transfer,[:direct,:httpgw,:connect,:node],"type of transfer agent")
49
- options.add_opt_list(:progress,[:none,:native,:multi],"type of progress bar")
50
- options.set_option(:transfer,:direct)
51
- options.set_option(:src_type,:list)
52
- options.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
53
- options.parse_options!
40
+ @opt_mgr.set_obj_attr(:ts,self,:option_transfer_spec)
41
+ @opt_mgr.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:ts,:optional)}")
42
+ @opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume,:optional)}")
43
+ @opt_mgr.add_opt_simple(:to_folder,'destination folder for downloaded files')
44
+ @opt_mgr.add_opt_simple(:sources,'list of source files (see doc)')
45
+ @opt_mgr.add_opt_simple(:transfer_info,'parameters for transfer agent')
46
+ @opt_mgr.add_opt_list(:src_type,[:list,:pair],'type of file list')
47
+ @opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,'type of transfer agent')
48
+ @opt_mgr.add_opt_list(:progress,[:none,:native,:multi],'type of progress bar')
49
+ @opt_mgr.set_option(:transfer,:direct)
50
+ @opt_mgr.set_option(:src_type,:list)
51
+ @opt_mgr.set_option(:progress,:native) # use native ascp progress bar as it is more reliable
52
+ @opt_mgr.parse_options!
54
53
  end
55
54
 
56
- def options; @env[:options];end
57
-
58
- def config; @env[:config];end
59
-
60
55
  def option_transfer_spec; @transfer_spec_cmdline; end
61
56
 
62
57
  # multiple option are merged
@@ -64,12 +59,12 @@ END_OF_TEMPLATE
64
59
 
65
60
  def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
66
61
 
67
- def set_agent_instance(instance)
62
+ def agent_instance=(instance)
68
63
  @agent=instance
69
64
  @agent.add_listener(Listener::Logger.new)
70
65
  # use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
71
- if options.get_option(:progress,:mandatory).eql?(:multi) or
72
- (options.get_option(:progress,:mandatory).eql?(:native) and !options.get_option(:transfer,:mandatory).eql?(:direct))
66
+ if @opt_mgr.get_option(:progress,:mandatory).eql?(:multi) ||
67
+ (@opt_mgr.get_option(:progress,:mandatory).eql?(:native) && !instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
73
68
  @agent.add_listener(@progress_listener)
74
69
  end
75
70
  end
@@ -77,57 +72,25 @@ END_OF_TEMPLATE
77
72
  # analyze options and create new agent if not already created or set
78
73
  def set_agent_by_options
79
74
  return nil unless @agent.nil?
80
- agent_type=options.get_option(:transfer,:mandatory)
81
- case agent_type
82
- when :direct
83
- agent_options=options.get_option(:transfer_info,:optional)
84
- agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
85
- new_agent=Fasp::Local.new(agent_options)
86
- new_agent.quiet=false if options.get_option(:progress,:mandatory).eql?(:native)
87
- when :httpgw
88
- httpgw_config=options.get_option(:transfer_info,:mandatory)
89
- new_agent=Fasp::HttpGW.new(httpgw_config)
90
- when :connect
91
- new_agent=Fasp::Connect.new
92
- when :node
93
- # way for code to setup alternate node api in advance
94
- # support: @preset:<name>
95
- # support extended values
96
- node_config=options.get_option(:transfer_info,:optional)
97
- # if not specified: use default node
98
- if node_config.nil?
99
- param_set_name=config.get_plugin_default_config_name(:node)
100
- raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
101
- node_config=config.preset_by_name(param_set_name)
102
- end
103
- Log.log.debug("node=#{node_config}")
104
- raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" unless node_config.is_a?(Hash)
105
- # here, node_config is a Hash
106
- node_config=node_config.symbolize_keys
107
- # Check mandatory params
108
- [:url,:username,:password].each { |k| raise CliBadArgument,"missing parameter [#{k}] in node specification: #{node_config}" unless node_config.has_key?(k) }
109
- if node_config[:password].match(/^Bearer /)
110
- node_api=Rest.new({
111
- base_url: node_config[:url],
112
- headers: {
113
- 'X-Aspera-AccessKey'=>node_config[:username],
114
- 'Authorization' =>node_config[:password]}})
115
- else
116
- node_api=Rest.new({
117
- base_url: node_config[:url],
118
- auth: {
119
- type: :basic,
120
- username: node_config[:username],
121
- password: node_config[:password]
122
- }})
123
- end
124
- new_agent=Fasp::Node.new(node_api)
125
- # add root id if it's an access key
126
- new_agent.options={root_id: node_config[:root_id]} if node_config.has_key?(:root_id)
127
- else
128
- raise "Unexpected transfer agent type: #{agent_type}"
75
+ agent_type=@opt_mgr.get_option(:transfer,:mandatory)
76
+ require "aspera/fasp/agent_#{agent_type}"
77
+ agent_options=@opt_mgr.get_option(:transfer_info,:optional)
78
+ raise CliBadArgument,"the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), use either @json:<json> or @preset:<parameter set name>" unless [Hash,NilClass].include?(agent_options.class)
79
+ # special case
80
+ if agent_type.eql?(:node) && agent_options.nil?
81
+ param_set_name=@config.get_plugin_default_config_name(:node)
82
+ raise CliBadArgument,"No default node configured, Please specify --#{:transfer_info.to_s.gsub('_','-')}" if param_set_name.nil?
83
+ agent_options=@config.preset_by_name(param_set_name)
84
+ end
85
+ # special case
86
+ if agent_type.eql?(:direct) && @opt_mgr.get_option(:progress,:mandatory).eql?(:native)
87
+ agent_options={} if agent_options.nil?
88
+ agent_options[:quiet]=false
129
89
  end
130
- set_agent_instance(new_agent)
90
+ agent_options=agent_options.symbolize_keys if agent_options.is_a?(Hash)
91
+ # get agent instance
92
+ new_agent=Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
93
+ self.agent_instance=new_agent
131
94
  return nil
132
95
  end
133
96
 
@@ -135,14 +98,14 @@ END_OF_TEMPLATE
135
98
  # sets default if needed
136
99
  # param: 'send' or 'receive'
137
100
  def destination_folder(direction)
138
- dest_folder=options.get_option(:to_folder,:optional)
139
- return dest_folder unless dest_folder.nil?
101
+ dest_folder=@opt_mgr.get_option(:to_folder,:optional)
102
+ return File.expand_path(dest_folder) unless dest_folder.nil?
140
103
  dest_folder=@transfer_spec_cmdline['destination_root']
141
104
  return dest_folder unless dest_folder.nil?
142
105
  # default: / on remote, . on local
143
106
  case direction.to_s
144
- when 'send';dest_folder='/'
145
- when 'receive';dest_folder='.'
107
+ when Fasp::TransferSpec::DIRECTION_SEND then dest_folder='/'
108
+ when Fasp::TransferSpec::DIRECTION_RECEIVE then dest_folder='.'
146
109
  else raise "wrong direction: #{direction}"
147
110
  end
148
111
  return dest_folder
@@ -150,7 +113,7 @@ END_OF_TEMPLATE
150
113
 
151
114
  # This is how the list of files to be transfered is specified
152
115
  # get paths suitable for transfer spec from command line
153
- # @return {:source=>(mandatory), :destination=>(optional)}
116
+ # @return {source: (mandatory), destination: (optional)}
154
117
  # computation is done only once, cache is kept in @transfer_paths
155
118
  def ts_source_paths
156
119
  # return cache if set
@@ -158,37 +121,37 @@ END_OF_TEMPLATE
158
121
  # start with lower priority : get paths from transfer spec on command line
159
122
  @transfer_paths=@transfer_spec_cmdline['paths'] if @transfer_spec_cmdline.has_key?('paths')
160
123
  # is there a source list option ?
161
- file_list=options.get_option(:sources,:optional)
124
+ file_list=@opt_mgr.get_option(:sources,:optional)
162
125
  case file_list
163
126
  when nil,FILE_LIST_FROM_ARGS
164
- Log.log.debug("getting file list as parameters")
127
+ Log.log.debug('getting file list as parameters')
165
128
  # get remaining arguments
166
- file_list=options.get_next_argument("source file list",:multiple)
167
- raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) or file_list.empty?
129
+ file_list=@opt_mgr.get_next_argument('source file list',:multiple)
130
+ raise CliBadArgument,"specify at least one file on command line or use --sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) || file_list.empty?
168
131
  when FILE_LIST_FROM_TRANSFER_SPEC
169
- Log.log.debug("assume list provided in transfer spec")
170
- special_case_direct_with_list=options.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
171
- raise CliBadArgument,"transfer spec on command line must have sources" if @transfer_paths.nil? and !special_case_direct_with_list
132
+ Log.log.debug('assume list provided in transfer spec')
133
+ special_case_direct_with_list=@opt_mgr.get_option(:transfer,:mandatory).eql?(:direct) and Fasp::Parameters.ts_has_file_list(@transfer_spec_cmdline)
134
+ raise CliBadArgument,'transfer spec on command line must have sources' if @transfer_paths.nil? && !special_case_direct_with_list
172
135
  # here we assume check of sources is made in transfer agent
173
136
  return @transfer_paths
174
137
  when Array
175
- Log.log.debug("getting file list as extended value")
176
- raise CliBadArgument,"sources must be a Array of String" if !file_list.select{|f|!f.is_a?(String)}.empty?
138
+ Log.log.debug('getting file list as extended value')
139
+ raise CliBadArgument,'sources must be a Array of String' if !file_list.reject{|f|f.is_a?(String)}.empty?
177
140
  else
178
141
  raise CliBadArgument,"sources must be a Array, not #{file_list.class}"
179
142
  end
180
143
  # here, file_list is an Array or String
181
144
  if !@transfer_paths.nil?
182
- Log.log.warn("--sources overrides paths from --ts")
145
+ Log.log.warn('--sources overrides paths from --ts')
183
146
  end
184
- case options.get_option(:src_type,:mandatory)
147
+ case @opt_mgr.get_option(:src_type,:mandatory)
185
148
  when :list
186
149
  # when providing a list, just specify source
187
150
  @transfer_paths=file_list.map{|i|{'source'=>i}}
188
151
  when :pair
189
152
  raise CliBadArgument,"When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
190
153
  @transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
191
- else raise "Unsupported src_type"
154
+ else raise 'Unsupported src_type'
192
155
  end
193
156
  Log.log.debug("paths=#{@transfer_paths}")
194
157
  return @transfer_paths
@@ -201,14 +164,14 @@ END_OF_TEMPLATE
201
164
  # other options are carried to specific agent
202
165
  def start(transfer_spec,tr_opts)
203
166
  # check parameters
204
- raise "transfer_spec must be hash" unless transfer_spec.is_a?(Hash)
205
- raise "tr_opts must be hash" unless tr_opts.is_a?(Hash)
167
+ raise 'transfer_spec must be hash' unless transfer_spec.is_a?(Hash)
168
+ raise 'tr_opts must be hash' unless tr_opts.is_a?(Hash)
206
169
  # process :src option
207
170
  case transfer_spec['direction']
208
- when 'receive'
171
+ when Fasp::TransferSpec::DIRECTION_RECEIVE
209
172
  # init default if required in any case
210
173
  @transfer_spec_cmdline['destination_root']||=destination_folder(transfer_spec['direction'])
211
- when 'send'
174
+ when Fasp::TransferSpec::DIRECTION_SEND
212
175
  case tr_opts[:src]
213
176
  when :direct
214
177
  # init default if required
@@ -232,18 +195,19 @@ END_OF_TEMPLATE
232
195
 
233
196
  transfer_spec.merge!(@transfer_spec_cmdline)
234
197
  # create transfer agent
235
- self.set_agent_by_options
198
+ set_agent_by_options
236
199
  Log.log.debug("transfer agent is a #{@agent.class}")
237
200
  @agent.start_transfer(transfer_spec,tr_opts)
201
+ # list of : :success or error message
238
202
  result=@agent.wait_for_transfers_completion
239
203
  @progress_listener.reset
240
- Fasp::Manager.validate_status_list(result)
204
+ Fasp::AgentBase.validate_status_list(result)
241
205
  send_email_transfer_notification(transfer_spec,result)
242
206
  return result
243
207
  end
244
208
 
245
209
  def send_email_transfer_notification(transfer_spec,statuses)
246
- return if options.get_option(:notif_to,:optional).nil?
210
+ return if @opt_mgr.get_option(:notif_to,:optional).nil?
247
211
  global_status=self.class.session_status(statuses)
248
212
  email_vars={
249
213
  global_transfer_status: global_status,
@@ -251,13 +215,13 @@ END_OF_TEMPLATE
251
215
  body: "Transfer is: #{global_status}",
252
216
  ts: transfer_spec
253
217
  }
254
- @env[:config].send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
218
+ @config.send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
255
219
  end
256
220
 
257
221
  # @return :success if all sessions statuses returned by "start" are success
258
222
  # else return the first error exception object
259
223
  def self.session_status(statuses)
260
- error_statuses=statuses.select{|i|!i.eql?(:success)}
224
+ error_statuses=statuses.reject{|i|i.eql?(:success)}
261
225
  return :success if error_statuses.empty?
262
226
  return error_statuses.first
263
227
  end
@@ -266,7 +230,6 @@ END_OF_TEMPLATE
266
230
  def shutdown
267
231
  @agent.shutdown if @agent.respond_to?(:shutdown)
268
232
  end
269
-
270
233
  end
271
234
  end
272
235
  end
@@ -1,5 +1,6 @@
1
+ # frozen_string_literal: true
1
2
  module Aspera
2
3
  module Cli
3
- VERSION = "4.4.0"
4
+ VERSION = '4.7.0'
4
5
  end
5
6
  end
data/lib/aspera/colors.rb CHANGED
@@ -1,43 +1,51 @@
1
+ # frozen_string_literal: true
1
2
  # simple vt100 colors
2
3
  class String
3
- private
4
- def self.vtcmd(code);"\e[#{code}m";end
4
+ class<<self
5
+ private
6
+
7
+ def vtcmd(code);"\e[#{code}m";end
8
+ end
5
9
  # see https://en.wikipedia.org/wiki/ANSI_escape_code
6
10
  # symbol is the method name added to String
7
11
  # it adds control chars to set color (and reset at the end).
8
12
  VTSTYLES = {
9
- :bold=>1,
10
- :italic=>3,
11
- :underline=>4,
12
- :blink=>5,
13
- :reverse_color=>7,
14
- :black=>30,
15
- :red=>31,
16
- :green=>32,
17
- :brown=>33,
18
- :blue=>34,
19
- :magenta=>35,
20
- :cyan=>36,
21
- :gray=>37,
22
- :bg_black=>40,
23
- :bg_red=>41,
24
- :bg_green=>42,
25
- :bg_brown=>43,
26
- :bg_blue=>44,
27
- :bg_magenta=>45,
28
- :bg_cyan=>46,
29
- :bg_gray=>47,
13
+ bold: 1,
14
+ italic: 3,
15
+ underline: 4,
16
+ blink: 5,
17
+ reverse_color: 7,
18
+ black: 30,
19
+ red: 31,
20
+ green: 32,
21
+ brown: 33,
22
+ blue: 34,
23
+ magenta: 35,
24
+ cyan: 36,
25
+ gray: 37,
26
+ bg_black: 40,
27
+ bg_red: 41,
28
+ bg_green: 42,
29
+ bg_brown: 43,
30
+ bg_blue: 44,
31
+ bg_magenta: 45,
32
+ bg_cyan: 46,
33
+ bg_gray: 47
30
34
  }
31
35
  private_constant :VTSTYLES
32
36
  # defines methods to String, one per entry in VTSTYLES
33
37
  VTSTYLES.each do |name,code|
34
- begin_seq=vtcmd(code)
35
- end_seq=vtcmd((code >= 10) ? 0 : code+20+(code.eql?(1)?1:0))
36
- if STDERR.tty?
38
+ if $stderr.tty?
39
+ begin_seq=vtcmd(code)
40
+ end_code=
41
+ if code >= 10 then 0
42
+ elsif code.eql?(1) then 22
43
+ else code+20
44
+ end
45
+ end_seq=vtcmd(end_code)
37
46
  define_method(name){"#{begin_seq}#{self}#{end_seq}"}
38
47
  else
39
48
  define_method(name){self}
40
49
  end
41
- public name
42
50
  end
43
51
  end
@@ -1,14 +1,15 @@
1
+ # frozen_string_literal: true
1
2
  module Aspera
2
3
  # helper class to build command line from a parameter list (key-value hash)
3
4
  # constructor takes hash: { 'param1':'value1', ...}
4
5
  # process_param is called repeatedly with all known parameters
5
6
  # add_env_args is called to get resulting param list and env var (also checks that all params were used)
6
7
  class CommandLineBuilder
7
- # transform yes/no to trye/false
8
+ # transform yes/no to true/false
8
9
  def self.yes_to_true(value)
9
10
  case value
10
- when 'yes'; return true
11
- when 'no'; return false
11
+ when 'yes' then return true
12
+ when 'no' then return false
12
13
  end
13
14
  raise "unsupported value: #{value}"
14
15
  end
@@ -22,13 +23,13 @@ module Aspera
22
23
  options[:mandatory]||=false
23
24
  options[:desc]||=''
24
25
  # by default : string, unless it's without arg
25
- if ! options.has_key?(:accepted_types)
26
+ if !options.has_key?(:accepted_types)
26
27
  options[:accepted_types]=options[:cltype].eql?(:opt_without_arg) ? :bool : :string
27
28
  end
28
29
  # single type is placed in array
29
30
  options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
30
- if !options.has_key?(:option_switch) and options.has_key?(:cltype) and [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
31
- options[:option_switch]='--'+param_name.to_s.gsub('_','-')
31
+ if !options.has_key?(:clswitch) && options.has_key?(:cltype) && [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
32
+ options[:clswitch]='--'+param_name.to_s.gsub('_','-')
32
33
  end
33
34
  end
34
35
  end
@@ -36,7 +37,7 @@ module Aspera
36
37
  private
37
38
 
38
39
  # clvarname : command line variable name
39
- def env_name(param_name,options)
40
+ def env_name(_param_name,options)
40
41
  return options[:clvarname]
41
42
  end
42
43
 
@@ -46,7 +47,7 @@ module Aspera
46
47
 
47
48
  # @param param_hash
48
49
  def initialize(param_hash,params_definition)
49
- @param_hash=param_hash # keep reference so that it can be modified by caller before calling `process_params`
50
+ @param_hash=param_hash # keep reference so that it can be modified by caller before calling `process_params`
50
51
  @params_definition=params_definition
51
52
  @result_env={}
52
53
  @result_args=[]
@@ -86,42 +87,53 @@ module Aspera
86
87
  # @param options : options for type
87
88
  def process_param(param_name,action=nil)
88
89
  options=@params_definition[param_name]
89
- action=options[:cltype] if action.nil?
90
90
  # should not happen
91
- raise "Internal error: ask processing of param #{param_name}" if options.nil?
91
+ if options.nil?
92
+ Log.log.warn("Unknown parameter #{param_name}")
93
+ return
94
+ end
95
+ action=options[:cltype] if action.nil?
92
96
  # check mandatory parameter (nil is valid value)
93
- raise Fasp::Error.new("mandatory parameter: #{param_name}") if options[:mandatory] and !@param_hash.has_key?(param_name)
97
+ raise Fasp::Error, "Missing mandatory parameter: #{param_name}" if options[:mandatory] && !@param_hash.has_key?(param_name)
94
98
  parameter_value=@param_hash[param_name]
99
+
95
100
  #parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
101
+
102
+ # Check parameter type
96
103
  expected_classes=options[:accepted_types].map do |s|
97
104
  case s
98
- when :string; String
99
- when :array; Array
100
- when :hash; Hash
101
- when :int; Integer
102
- when :bool; [TrueClass,FalseClass]
105
+ when :string then String
106
+ when :array then Array
107
+ when :hash then Hash
108
+ when :int then Integer
109
+ when :bool then [TrueClass,FalseClass]
103
110
  else raise "INTERNAL: unexpected value: #{s}"
104
111
  end
105
112
  end.flatten
106
- # check provided type
107
- raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or expected_classes.include?(parameter_value.class)
113
+ raise Fasp::Error,"#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, " unless parameter_value.nil? || expected_classes.include?(parameter_value.class)
108
114
  @used_param_names.push(param_name) unless action.eql?(:defer)
109
115
 
110
116
  # process only non-nil values
111
117
  return nil if parameter_value.nil?
112
118
 
113
- if options.has_key?(:translate_values)
119
+ # check that value is of an accepted type (string, int bool)
120
+ raise "Value #{parameter_value} is not allowed for #{param_name}" if options.has_key?(:enum) && !options[:enum].include?(parameter_value)
121
+
122
+ # convert some values if value on command line needs processing from value in structure
123
+ case options[:clconvert]
124
+ when Hash
114
125
  # translate using conversion table
115
- new_value=options[:translate_values][parameter_value]
116
- raise "unsupported value: #{parameter_value}" if new_value.nil?
126
+ new_value=options[:clconvert][parameter_value]
127
+ raise "unsupported value: #{parameter_value}, expect: #{options[:clconvert].keys.join(', ')}" if new_value.nil?
117
128
  parameter_value=new_value
118
- end
119
- raise "unsupported value: #{parameter_value}" unless options[:accepted_values].nil? or options[:accepted_values].include?(parameter_value)
120
- if options[:encode]
121
- # :encode has name of class with encoding method
122
- newvalue=Kernel.const_get(options[:encode]).send("encode_#{param_name}",parameter_value)
123
- raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
129
+ when String
130
+ # :clconvert has name of class and encoding method
131
+ convclass,convmethod=options[:clconvert].split('.')
132
+ newvalue=Kernel.const_get(convclass).send(convmethod,parameter_value)
133
+ raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}" if newvalue.nil?
124
134
  parameter_value=newvalue
135
+ when NilClass
136
+ else raise "not expected type for clconvert #{options[:clconvert].class} for #{param_name}"
125
137
  end
126
138
 
127
139
  case action
@@ -135,19 +147,21 @@ module Aspera
135
147
  when :opt_without_arg # if present and true : just add option without value
136
148
  add_param=false
137
149
  case parameter_value
138
- when false# nothing to put on command line, no creation by default
139
- when true; add_param=true
140
- else raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}")
150
+ when false then nil # nothing to put on command line, no creation by default
151
+ when true then add_param=true
152
+ else raise Fasp::Error, "unsupported #{param_name}: #{parameter_value}"
141
153
  end
142
- add_param=!add_param if options[:add_on_false]
143
- add_command_line_options([options[:option_switch]]) if add_param
154
+ add_param= !add_param if options[:add_on_false]
155
+ add_command_line_options([options[:clswitch]]) if add_param
144
156
  when :opt_with_arg # transform into command line option with value
145
157
  #parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
146
158
  parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
147
159
  # if transfer_spec value is an array, applies option many times
148
- parameter_value.each{|v|add_command_line_options([options[:option_switch],v])}
160
+ parameter_value.each{|v|add_command_line_options([options[:clswitch],v])}
161
+ when NilClass
162
+ Log.log.debug("Ignoring parameter: #{param_name}")
149
163
  else
150
- raise "Error"
164
+ raise "ERROR: unknown action: #{action}/#{action.class}"
151
165
  end
152
166
  end
153
167
  end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'aspera/log'
2
3
  require 'aspera/rest'
3
4
  require 'xmlsimple'
@@ -11,25 +12,28 @@ module Aspera
11
12
  @auth_url=auth_url
12
13
  @api_key=api_key
13
14
  s3_api=Aspera::Rest.new({
14
- :base_url => storage_endpoint,
15
- :not_auth_codes => ['401','403'], # error codes when not authorized
16
- :headers => {'ibm-service-instance-id' => instance_id},
17
- :auth => {
18
- :type => :oauth2,
19
- :base_url => @auth_url,
20
- :grant => :ibm_apikey,
21
- :api_key => @api_key
22
- }})
15
+ base_url: storage_endpoint,
16
+ not_auth_codes: ['401','403'], # error codes when not authorized
17
+ headers: {'ibm-service-instance-id' => instance_id},
18
+ auth: {
19
+ type: :oauth2,
20
+ base_url: @auth_url,
21
+ crtype: :generic,
22
+ generic: {
23
+ grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
24
+ response_type: 'cloud_iam',
25
+ apikey: @api_key
26
+ }}})
23
27
  # read FASP connection information for bucket
24
- xml_result_text=s3_api.call({:operation=>'GET',:subpath=>bucket_name,:headers=>{'Accept'=>'application/xml'},:url_params=>{'faspConnectionInfo'=>nil}})[:http].body
28
+ xml_result_text=s3_api.call({operation: 'GET',subpath: bucket_name,headers: {'Accept'=>'application/xml'},url_params: {'faspConnectionInfo'=>nil}})[:http].body
25
29
  ats_info=XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
26
30
  Aspera::Log.dump('ats_info',ats_info)
27
31
  super({
28
- :base_url => ats_info['ATSEndpoint'],
29
- :auth => {
30
- :type => :basic,
31
- :username => ats_info['AccessKey']['Id'],
32
- :password => ats_info['AccessKey']['Secret']}})
32
+ base_url: ats_info['ATSEndpoint'],
33
+ auth: {
34
+ type: :basic,
35
+ username: ats_info['AccessKey']['Id'],
36
+ password: ats_info['AccessKey']['Secret']}})
33
37
  # prepare transfer spec addition
34
38
  @add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
35
39
  'type' => 'token',
@@ -42,12 +46,16 @@ module Aspera
42
46
  def generate_token
43
47
  # OAuth API to get delegated token
44
48
  delegated_oauth=Oauth.new({
45
- :type => :oauth2,
46
- :base_url => @auth_url,
47
- :grant => :delegated_refresh,
48
- :api_key => @api_key,
49
- :token_field=> TOKEN_FIELD
50
- })
49
+ type: :oauth2,
50
+ base_url: @auth_url,
51
+ token_field: TOKEN_FIELD,
52
+ crtype: :generic,
53
+ generic: {
54
+ grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
55
+ response_type: 'delegated_refresh_token',
56
+ apikey: @api_key,
57
+ receiver_client_ids: 'aspera_ats'
58
+ }})
51
59
  # get delagated token to be placed in rest call header and in transfer tags
52
60
  @add_ts['tags']['aspera']['node']['storage_credentials']['token'][TOKEN_FIELD]=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
53
61
  @params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'aspera/log'
2
3
  require 'singleton'
3
4
 
@@ -6,8 +7,8 @@ module Aspera
6
7
  class DataRepository
7
8
  include Singleton
8
9
  # get binary value from data repository
9
- def get_bin(id)
10
- File.read(File.join(File.expand_path(File.dirname(__FILE__)),'data',id.to_s),mode: 'rb')
10
+ def data(id)
11
+ File.read(File.join(__dir__,'data',id.to_s),mode: 'rb')
11
12
  end
12
13
  end
13
14
  end