aspera-cli 4.15.0 → 4.16.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +29 -3
- data/CHANGELOG.md +292 -228
- data/CONTRIBUTING.md +69 -18
- data/README.md +1102 -952
- data/bin/ascli +13 -31
- data/bin/asession +3 -1
- data/examples/dascli +2 -2
- data/lib/aspera/aoc.rb +28 -33
- data/lib/aspera/ascmd.rb +3 -6
- data/lib/aspera/assert.rb +45 -0
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formatter.rb +26 -13
- data/lib/aspera/cli/hints.rb +4 -3
- data/lib/aspera/cli/main.rb +16 -3
- data/lib/aspera/cli/manager.rb +45 -36
- data/lib/aspera/cli/plugin.rb +20 -13
- data/lib/aspera/cli/plugins/aoc.rb +103 -73
- data/lib/aspera/cli/plugins/ats.rb +4 -3
- data/lib/aspera/cli/plugins/config.rb +114 -119
- data/lib/aspera/cli/plugins/cos.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +23 -19
- data/lib/aspera/cli/plugins/faspex5.rb +75 -43
- data/lib/aspera/cli/plugins/node.rb +28 -15
- data/lib/aspera/cli/plugins/orchestrator.rb +4 -2
- data/lib/aspera/cli/plugins/preview.rb +9 -7
- data/lib/aspera/cli/plugins/server.rb +6 -3
- data/lib/aspera/cli/plugins/shares.rb +30 -26
- data/lib/aspera/cli/sync_actions.rb +9 -9
- data/lib/aspera/cli/transfer_agent.rb +21 -14
- data/lib/aspera/cli/transfer_progress.rb +2 -3
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +13 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/coverage.rb +22 -0
- data/lib/aspera/data_repository.rb +33 -2
- data/lib/aspera/environment.rb +4 -2
- data/lib/aspera/fasp/{agent_aspera.rb → agent_alpha.rb} +29 -39
- data/lib/aspera/fasp/agent_base.rb +17 -7
- data/lib/aspera/fasp/agent_direct.rb +88 -84
- data/lib/aspera/fasp/agent_httpgw.rb +4 -3
- data/lib/aspera/fasp/agent_node.rb +3 -2
- data/lib/aspera/fasp/agent_trsdk.rb +79 -37
- data/lib/aspera/fasp/installation.rb +51 -12
- data/lib/aspera/fasp/management.rb +11 -6
- data/lib/aspera/fasp/parameters.rb +53 -47
- data/lib/aspera/fasp/resume_policy.rb +7 -5
- data/lib/aspera/fasp/sync.rb +273 -0
- data/lib/aspera/fasp/transfer_spec.rb +10 -8
- data/lib/aspera/fasp/uri.rb +2 -2
- data/lib/aspera/faspex_gw.rb +11 -8
- data/lib/aspera/faspex_postproc.rb +6 -5
- data/lib/aspera/id_generator.rb +3 -1
- data/lib/aspera/json_rpc.rb +10 -8
- data/lib/aspera/keychain/encrypted_hash.rb +46 -11
- data/lib/aspera/keychain/macos_security.rb +15 -13
- data/lib/aspera/log.rb +4 -3
- data/lib/aspera/nagios.rb +7 -2
- data/lib/aspera/node.rb +17 -16
- data/lib/aspera/node_simulator.rb +214 -0
- data/lib/aspera/oauth.rb +22 -19
- data/lib/aspera/persistency_action_once.rb +13 -14
- data/lib/aspera/persistency_folder.rb +3 -2
- data/lib/aspera/preview/file_types.rb +53 -267
- data/lib/aspera/preview/generator.rb +7 -5
- data/lib/aspera/preview/terminal.rb +14 -5
- data/lib/aspera/preview/utils.rb +8 -7
- data/lib/aspera/proxy_auto_config.rb +6 -3
- data/lib/aspera/rest.rb +29 -13
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/rest_errors_aspera.rb +2 -0
- data/lib/aspera/secret_hider.rb +5 -2
- data/lib/aspera/ssh.rb +10 -8
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/web_server_simple.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +96 -45
- metadata.gz.sig +0 -0
- data/lib/aspera/sync.rb +0 -219
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/cli/plugins/node'
|
4
|
-
|
4
|
+
require 'aspera/assert'
|
5
5
|
module Aspera
|
6
6
|
module Cli
|
7
7
|
module Plugins
|
@@ -49,8 +49,6 @@ module Aspera
|
|
49
49
|
|
50
50
|
def initialize(env)
|
51
51
|
super(env)
|
52
|
-
options.declare(:type, 'Type of user/group for operations', values: %i[any local ldap saml], default: :any)
|
53
|
-
options.parse_options!
|
54
52
|
end
|
55
53
|
|
56
54
|
SAML_IMPORT_MANDATORY = %w[id name_id].freeze
|
@@ -84,63 +82,69 @@ module Aspera
|
|
84
82
|
return Node.new(@agents, api: api_shares_node).execute_action(repo_command)
|
85
83
|
when :admin
|
86
84
|
api_shares_admin = basic_auth_api('api/v1')
|
87
|
-
admin_command = options.get_next_command(%i[user group
|
85
|
+
admin_command = options.get_next_command(%i[node share transfer_settings user group].freeze)
|
88
86
|
case admin_command
|
89
87
|
when :node
|
90
88
|
return entity_action(api_shares_admin, 'data/nodes')
|
89
|
+
when :share
|
90
|
+
share_command = options.get_next_command(%i[user_permissions group_permissions].concat(Plugin::ALL_OPS))
|
91
|
+
case share_command
|
92
|
+
when *Plugin::ALL_OPS
|
93
|
+
return entity_command(share_command, api_shares_admin, 'data/shares')
|
94
|
+
# return {type: :object_list, data: all_shares, fields: %w[id name status status_message]}
|
95
|
+
when :user_permissions, :group_permissions
|
96
|
+
share_id = instance_identifier
|
97
|
+
return entity_action(api_shares_admin, "data/shares/#{share_id}/#{share_command}")
|
98
|
+
end
|
99
|
+
when :transfer_settings
|
100
|
+
xfer_settings_command = options.get_next_command(%i[show modify])
|
101
|
+
return entity_command(xfer_settings_command, api_shares_admin, 'data/transfer_settings', is_singleton: true)
|
91
102
|
when :user, :group
|
92
103
|
entity_type = admin_command
|
93
|
-
entities_location = options.
|
104
|
+
entities_location = options.get_next_command(%i[all local ldap saml])
|
94
105
|
entities_path = "data/#{entities_location}_#{entity_type}s"
|
95
106
|
entity_action = nil
|
96
107
|
case entities_location
|
97
|
-
when :
|
108
|
+
when :all
|
98
109
|
entities_path = "data/#{entity_type}s"
|
99
110
|
entity_action = %i[list show delete]
|
100
111
|
entity_action.concat(USR_GRP_SETTINGS)
|
101
112
|
entity_action.push(:users) if entity_type.eql?(:group)
|
102
113
|
entity_action.freeze
|
103
114
|
when :local
|
104
|
-
entity_action = %i[list show create modify
|
115
|
+
entity_action = %i[list show delete create modify].freeze
|
105
116
|
when :ldap
|
106
117
|
entity_action = %i[add].freeze
|
107
118
|
when :saml
|
108
119
|
entity_action = %i[import].freeze
|
109
120
|
end
|
110
121
|
entity_verb = options.get_next_command(entity_action)
|
111
|
-
# entity_path = "#{entities_path}/#{instance_identifier}" if %i[app_authorizations share_permissions].include?(entity_verb)
|
112
122
|
case entity_verb
|
113
|
-
when *Plugin::ALL_OPS
|
123
|
+
when *Plugin::ALL_OPS # list, show, delete, create, modify
|
114
124
|
display_fields = entity_type.eql?(:user) ? %w[id username first_name last_name email] : nil
|
115
|
-
display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:
|
125
|
+
display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:all)
|
116
126
|
return entity_command(entity_verb, api_shares_admin, entities_path, display_fields: display_fields)
|
117
|
-
when
|
127
|
+
when *USR_GRP_SETTINGS # transfer_settings, app_authorizations, share_permissions
|
128
|
+
group_id = instance_identifier
|
129
|
+
entities_path = "#{entities_path}/#{group_id}/#{entity_verb}"
|
130
|
+
return entity_action(api_shares_admin, entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
|
131
|
+
when :import # saml
|
118
132
|
return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
|
119
133
|
entity_parameters = entity_parameters.transform_keys{|k|k.gsub(/\s+/, '_').downcase}
|
120
|
-
|
134
|
+
assert_type(entity_parameters, Hash)
|
121
135
|
SAML_IMPORT_MANDATORY.each{|p|raise "missing mandatory field: #{p}" if entity_parameters[p].nil?}
|
122
136
|
entity_parameters.each_key do |p|
|
123
137
|
raise "unsupported field: #{p}, use: #{SAML_IMPORT_ALLOWED.join(',')}" unless SAML_IMPORT_ALLOWED.include?(p)
|
124
138
|
end
|
125
139
|
api_shares_admin.create("#{entities_path}/import", entity_parameters)[:data]
|
126
140
|
end
|
127
|
-
when :add
|
141
|
+
when :add # ldap
|
128
142
|
return do_bulk_operation(command: entity_verb, descr: "#{entity_type} name", values: String) do |entity_name|
|
129
143
|
api_shares_admin.create(entities_path, {entity_type=>entity_name})[:data]
|
130
144
|
end
|
131
|
-
when
|
132
|
-
|
133
|
-
|
134
|
-
return entity_action(api_shares_admin, entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
|
135
|
-
end
|
136
|
-
when :share
|
137
|
-
share_command = options.get_next_command(%i[user_permissions group_permissions].concat(Plugin::ALL_OPS))
|
138
|
-
case share_command
|
139
|
-
when *Plugin::ALL_OPS
|
140
|
-
return entity_command(share_command, api_shares_admin, 'data/shares')
|
141
|
-
# return {type: :object_list, data: all_shares, fields: %w[id name status status_message]}
|
142
|
-
when :user_permissions, :group_permissions
|
143
|
-
return entity_action(api_shares_admin, "data/shares/#{instance_identifier}/#{share_command}")
|
145
|
+
when :users # group
|
146
|
+
raise "TODO, not implemented"
|
147
|
+
else error_unexpected_value(entity_verb)
|
144
148
|
end
|
145
149
|
end
|
146
150
|
end
|
@@ -1,13 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'aspera/sync'
|
3
|
+
require 'aspera/fasp/sync'
|
4
|
+
require 'aspera/assert'
|
4
5
|
|
5
6
|
module Aspera
|
6
7
|
module Cli
|
7
8
|
# Module for sync actions
|
8
9
|
module SyncActions
|
9
10
|
SIMPLE_ARGUMENTS_SYNC = {
|
10
|
-
direction: Aspera::Sync::DIRECTIONS,
|
11
|
+
direction: Aspera::Fasp::Sync::DIRECTIONS,
|
11
12
|
local_dir: String,
|
12
13
|
remote_dir: String
|
13
14
|
}.stringify_keys.freeze
|
@@ -19,8 +20,7 @@ module Aspera
|
|
19
20
|
end
|
20
21
|
|
21
22
|
def execute_sync_action(&block)
|
22
|
-
|
23
|
-
raise 'Internal Error: No block given' unless block
|
23
|
+
assert(block){'No block given'}
|
24
24
|
command = options.get_next_command(%i[start admin])
|
25
25
|
# try to get 3 arguments as simple arguments
|
26
26
|
case command
|
@@ -45,13 +45,13 @@ module Aspera
|
|
45
45
|
:sync_info,
|
46
46
|
mandatory: false,
|
47
47
|
default: {'sessions' => [{'name' => File.basename(simple_session_args['local_dir'])}]})
|
48
|
-
|
49
|
-
|
50
|
-
|
48
|
+
assert_type(async_params, Hash){'sync_info'}
|
49
|
+
assert_type(async_params['sessions'], Array){'sync_info[sessions]'}
|
50
|
+
assert_type(async_params['sessions'].first, Hash){'sync_info[sessions][0]'}
|
51
51
|
async_params['sessions'].first.merge!(simple_session_args)
|
52
52
|
end
|
53
53
|
Log.log.debug{Log.dump('async_params', async_params)}
|
54
|
-
Aspera::Sync.start(async_params, &block)
|
54
|
+
Aspera::Fasp::Sync.start(async_params, &block)
|
55
55
|
return Main.result_success
|
56
56
|
when :admin
|
57
57
|
command2 = options.get_next_command([:status])
|
@@ -59,7 +59,7 @@ module Aspera
|
|
59
59
|
when :status
|
60
60
|
sync_session_name = options.get_next_argument('name of sync session', mandatory: false, type: String)
|
61
61
|
async_params = options.get_option(:sync_info, mandatory: true)
|
62
|
-
return {type: :single_object, data: Aspera::Sync.admin_status(async_params, sync_session_name)}
|
62
|
+
return {type: :single_object, data: Aspera::Fasp::Sync.admin_status(async_params, sync_session_name)}
|
63
63
|
end # command2
|
64
64
|
end # command
|
65
65
|
end # execute_action
|
@@ -1,7 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/fasp/agent_base'
|
3
4
|
require 'aspera/fasp/transfer_spec'
|
4
5
|
require 'aspera/cli/info'
|
6
|
+
require 'aspera/log'
|
7
|
+
require 'aspera/assert'
|
5
8
|
|
6
9
|
module Aspera
|
7
10
|
module Cli
|
@@ -28,7 +31,7 @@ module Aspera
|
|
28
31
|
:FILE_LIST_FROM_TRANSFER_SPEC,
|
29
32
|
:FILE_LIST_OPTIONS,
|
30
33
|
:DEFAULT_TRANSFER_NOTIFY_TEMPLATE
|
31
|
-
TRANSFER_AGENTS =
|
34
|
+
TRANSFER_AGENTS = Fasp::AgentBase.agent_list.freeze
|
32
35
|
|
33
36
|
class << self
|
34
37
|
# @return :success if all sessions statuses returned by "start" are success
|
@@ -65,7 +68,7 @@ module Aspera
|
|
65
68
|
|
66
69
|
# multiple option are merged
|
67
70
|
def option_transfer_spec=(value)
|
68
|
-
|
71
|
+
assert_type(value, Hash){'ts'}
|
69
72
|
@transfer_spec_command_line.deep_merge!(value)
|
70
73
|
end
|
71
74
|
|
@@ -92,8 +95,8 @@ module Aspera
|
|
92
95
|
end
|
93
96
|
|
94
97
|
# analyze options and create new agent if not already created or set
|
95
|
-
def
|
96
|
-
return
|
98
|
+
def agent_instance
|
99
|
+
return @agent unless @agent.nil?
|
97
100
|
agent_type = @opt_mgr.get_option(:transfer, mandatory: true)
|
98
101
|
# agent plugin is loaded on demand to avoid loading unnecessary dependencies
|
99
102
|
require "aspera/fasp/agent_#{agent_type}"
|
@@ -117,7 +120,8 @@ module Aspera
|
|
117
120
|
# get agent instance
|
118
121
|
new_agent = Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
|
119
122
|
self.agent_instance = new_agent
|
120
|
-
|
123
|
+
Log.log.debug{"transfer agent is a #{@agent.class}"}
|
124
|
+
return @agent
|
121
125
|
end
|
122
126
|
|
123
127
|
# return destination folder for transfers
|
@@ -133,7 +137,7 @@ module Aspera
|
|
133
137
|
case direction.to_s
|
134
138
|
when Fasp::TransferSpec::DIRECTION_SEND then dest_folder = '/'
|
135
139
|
when Fasp::TransferSpec::DIRECTION_RECEIVE then dest_folder = '.'
|
136
|
-
else
|
140
|
+
else error_unexpected_value(direction)
|
137
141
|
end
|
138
142
|
return dest_folder
|
139
143
|
end
|
@@ -181,14 +185,15 @@ module Aspera
|
|
181
185
|
if !@transfer_paths.nil?
|
182
186
|
Log.log.warn('--sources overrides paths from --ts')
|
183
187
|
end
|
184
|
-
|
188
|
+
source_type=@opt_mgr.get_option(:src_type, mandatory: true)
|
189
|
+
case source_type
|
185
190
|
when :list
|
186
191
|
# when providing a list, just specify source
|
187
192
|
@transfer_paths = file_list.map{|i|{'source' => i}}
|
188
193
|
when :pair
|
189
|
-
|
194
|
+
assert(file_list.length.even?, exception_class: Cli::BadArgument){"When using pair, provide an even number of paths: #{file_list.length}"}
|
190
195
|
@transfer_paths = file_list.each_slice(2).to_a.map{|s, d|{'source' => s, 'destination' => d}}
|
191
|
-
else
|
196
|
+
else error_unexpected_value(source_type)
|
192
197
|
end
|
193
198
|
Log.log.debug{"paths=#{@transfer_paths}"}
|
194
199
|
return @transfer_paths
|
@@ -199,7 +204,7 @@ module Aspera
|
|
199
204
|
# @param rest_token [Rest] if oauth token regeneration supported
|
200
205
|
def start(transfer_spec, rest_token: nil)
|
201
206
|
# check parameters
|
202
|
-
|
207
|
+
assert_type(transfer_spec, Hash){'transfer_spec'}
|
203
208
|
# process :src option
|
204
209
|
case transfer_spec['direction']
|
205
210
|
when Fasp::TransferSpec::DIRECTION_RECEIVE
|
@@ -223,12 +228,14 @@ module Aspera
|
|
223
228
|
@transfer_spec_command_line['paths'] = transfer_spec['paths'] || ts_source_paths
|
224
229
|
# updated transfer spec with command line
|
225
230
|
updated_ts(transfer_spec)
|
231
|
+
# if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
|
232
|
+
if transfer_spec['content_protection'].eql?('decrypt') && !transfer_spec.key?('content_protection_password')
|
233
|
+
transfer_spec['content_protection_password'] = @opt_mgr.prompt_user_input('content protection password', true)
|
234
|
+
end
|
226
235
|
# create transfer agent
|
227
|
-
|
228
|
-
Log.log.debug{"transfer agent is a #{@agent.class}"}
|
229
|
-
@agent.start_transfer(transfer_spec, token_regenerator: rest_token)
|
236
|
+
agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
|
230
237
|
# list of: :success or "error message string"
|
231
|
-
result =
|
238
|
+
result = agent_instance.wait_for_completion
|
232
239
|
send_email_transfer_notification(transfer_spec, result)
|
233
240
|
return result
|
234
241
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
4
5
|
require 'ruby-progressbar'
|
5
6
|
|
6
7
|
module Aspera
|
@@ -25,9 +26,7 @@ module Aspera
|
|
25
26
|
|
26
27
|
def event(session_id:, type:, info: nil)
|
27
28
|
Log.log.debug{"progress: #{type} #{session_id} #{info}"}
|
28
|
-
|
29
|
-
raise 'Internal error: session_id is nil'
|
30
|
-
end
|
29
|
+
assert(!session_id.nil? || type.eql?(:pre_start)){'session_id is nil'}
|
31
30
|
return if @completed
|
32
31
|
if @progress_bar.nil?
|
33
32
|
@progress_bar = ProgressBar.create(
|
data/lib/aspera/cli/version.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
3
5
|
module Aspera
|
4
6
|
# helper class to build command line from a parameter list (key-value hash)
|
5
7
|
# constructor takes hash: { 'param1':'value1', ...}
|
@@ -27,20 +29,20 @@ module Aspera
|
|
27
29
|
# Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
|
28
30
|
def normalize_description(full_description)
|
29
31
|
full_description.each do |name, options|
|
30
|
-
|
32
|
+
assert_type(options, Hash){name}
|
31
33
|
unsupported_keys = options.keys - OPTIONS_KEYS
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
34
|
+
assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
|
35
|
+
assert(options.key?(:cli)){"Missing key: cli for #{name}"}
|
36
|
+
assert_type(options[:cli], Hash){'Key: cli'}
|
37
|
+
assert(options[:cli].key?(:type)){'Missing key: cli.type'}
|
38
|
+
assert_values(options[:cli][:type], CLI_OPTION_TYPES){"Unsupported processing type for #{name}"}
|
37
39
|
# by default : optional
|
38
40
|
options[:mandatory] ||= false
|
39
41
|
options[:desc] ||= ''
|
40
42
|
options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
|
41
43
|
cli = options[:cli]
|
42
44
|
unsupported_cli_keys = cli.keys - CLI_KEYS
|
43
|
-
|
45
|
+
assert(unsupported_cli_keys.empty?){"Unsupported cli keys: #{unsupported_cli_keys}"}
|
44
46
|
# by default : string, unless it's without arg
|
45
47
|
options[:accepted_types] ||= options[:cli][:type].eql?(:opt_without_arg) ? :bool : :string
|
46
48
|
# single type is placed in array
|
@@ -121,7 +123,7 @@ module Aspera
|
|
121
123
|
when :hash then Hash
|
122
124
|
when :int then Integer
|
123
125
|
when :bool then [TrueClass, FalseClass]
|
124
|
-
else
|
126
|
+
else error_unexpected_value(type_symbol)
|
125
127
|
end
|
126
128
|
end.flatten
|
127
129
|
# check that value is of expected type
|
@@ -150,7 +152,7 @@ module Aspera
|
|
150
152
|
raise Fasp::Error, "unsupported #{name}: #{parameter_value}" if converted_value.nil?
|
151
153
|
parameter_value = converted_value
|
152
154
|
when NilClass
|
153
|
-
else
|
155
|
+
else error_unexpected_value(options[:cli][:convert].class)
|
154
156
|
end
|
155
157
|
|
156
158
|
case processing_type
|
@@ -159,14 +161,14 @@ module Aspera
|
|
159
161
|
when :ignore, :special # ignore this parameter or process later
|
160
162
|
return
|
161
163
|
when :envvar # set in env var
|
162
|
-
|
164
|
+
assert(options[:cli].key?(:variable)){'missing key: cli.variable'}
|
163
165
|
@result[:env][options[:cli][:variable]] = parameter_value
|
164
166
|
when :opt_without_arg # if present and true : just add option without value
|
165
167
|
add_param = false
|
166
168
|
case parameter_value
|
167
169
|
when false then nil # nothing to put on command line, no creation by default
|
168
170
|
when true then add_param = true
|
169
|
-
else
|
171
|
+
else error_unexpected_value(parameter_value){name}
|
170
172
|
end
|
171
173
|
add_param = !add_param if options[:add_on_false]
|
172
174
|
add_command_line_options([options[:cli][:switch]]) if add_param
|
data/lib/aspera/cos_node.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
4
5
|
require 'aspera/rest'
|
5
6
|
require 'aspera/oauth'
|
6
7
|
require 'xmlsimple'
|
@@ -10,9 +11,9 @@ module Aspera
|
|
10
11
|
class << self
|
11
12
|
def parameters_from_svc_credentials(service_credentials, bucket_region)
|
12
13
|
# check necessary contents
|
13
|
-
|
14
|
+
assert_type(service_credentials, Hash){'service_credentials'}
|
14
15
|
%w[apikey resource_instance_id endpoints].each do |field|
|
15
|
-
|
16
|
+
assert(service_credentials.key?(field)){"service_credentials must have a field: #{field}"}
|
16
17
|
end
|
17
18
|
Aspera::Log.dump('service_credentials', service_credentials)
|
18
19
|
# read endpoints from service provided in service credentials
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# coverage for tests
|
4
|
+
if ENV.key?('ENABLE_COVERAGE')
|
5
|
+
require 'simplecov'
|
6
|
+
require 'securerandom'
|
7
|
+
# compute gem source root based on this script location, assuming it is in bin/
|
8
|
+
# use dirname instead of gsub, in case folder separator is not /
|
9
|
+
development_root = 3.times.inject(File.realpath(__FILE__)) { |p, _| File.dirname(p) }
|
10
|
+
SimpleCov.root(development_root)
|
11
|
+
SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
|
12
|
+
# keep cache data for 1 day (must be longer that time to run the whole test suite)
|
13
|
+
SimpleCov.merge_timeout(86400)
|
14
|
+
SimpleCov.command_name(SecureRandom.uuid)
|
15
|
+
SimpleCov.at_exit do
|
16
|
+
original_file_descriptor = $stdout
|
17
|
+
$stdout.reopen(File.join(development_root, 'simplecov.log'))
|
18
|
+
SimpleCov.result.format!
|
19
|
+
$stdout.reopen(original_file_descriptor)
|
20
|
+
end
|
21
|
+
SimpleCov.start
|
22
|
+
end
|
@@ -1,15 +1,46 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'aspera/
|
3
|
+
require 'aspera/assert'
|
4
4
|
require 'singleton'
|
5
|
+
require 'openssl'
|
5
6
|
|
6
7
|
module Aspera
|
7
8
|
# a simple binary data repository
|
8
9
|
class DataRepository
|
9
10
|
include Singleton
|
11
|
+
# in same order as elements in folder
|
12
|
+
ELEMENTS = %i[dsa rsa uuid aspera.global-cli-client aspera.drive license]
|
13
|
+
START_INDEX = 1
|
14
|
+
DATA_FOLDER_NAME = 'data'
|
15
|
+
|
16
|
+
# decode data as expected as string
|
17
|
+
# @param name [Symbol] name of the data item
|
18
|
+
# @return [String] decoded data
|
19
|
+
def item(name)
|
20
|
+
index = ELEMENTS.index(name)
|
21
|
+
raise ArgumentError, "unknown data item #{name} (#{name.class})" unless index
|
22
|
+
raw_data = data(START_INDEX + index)
|
23
|
+
case name
|
24
|
+
when :dsa, :rsa
|
25
|
+
# generate PEM from DER
|
26
|
+
return OpenSSL::PKey.const_get(name.to_s.upcase).new(raw_data).to_pem
|
27
|
+
when :license
|
28
|
+
return Zlib::Inflate.inflate(raw_data)
|
29
|
+
when :uuid
|
30
|
+
return format('%08x-%04x-%04x-%04x-%04x%08x', *raw_data.unpack('NnnnnN'))
|
31
|
+
when :'aspera.global-cli-client', :'aspera.drive'
|
32
|
+
return Base64.urlsafe_encode64(raw_data)
|
33
|
+
else error_unexpected_value(name)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
private_constant :START_INDEX, :DATA_FOLDER_NAME
|
40
|
+
|
10
41
|
# get binary value from data repository
|
11
42
|
def data(id)
|
12
|
-
File.read(File.join(__dir__,
|
43
|
+
File.read(File.join(__dir__, DATA_FOLDER_NAME, id.to_s), mode: 'rb')
|
13
44
|
end
|
14
45
|
end
|
15
46
|
end
|
data/lib/aspera/environment.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
# cspell:ignore USERPROFILE HOMEDRIVE HOMEPATH LC_CTYPE msys aarch
|
4
4
|
require 'aspera/log'
|
5
|
+
require 'aspera/assert'
|
5
6
|
require 'rbconfig'
|
6
7
|
|
7
8
|
# cspell:words MEBI mswin bccwin
|
@@ -15,10 +16,11 @@ module Aspera
|
|
15
16
|
OS_AIX = :aix
|
16
17
|
OS_LIST = [OS_WINDOWS, OS_X, OS_LINUX, OS_AIX].freeze
|
17
18
|
CPU_X86_64 = :x86_64
|
19
|
+
CPU_ARM64 = :arm64
|
18
20
|
CPU_PPC64 = :ppc64
|
19
21
|
CPU_PPC64LE = :ppc64le
|
20
22
|
CPU_S390 = :s390
|
21
|
-
CPU_LIST = [CPU_X86_64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
|
23
|
+
CPU_LIST = [CPU_X86_64, CPU_ARM64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
|
22
24
|
|
23
25
|
BITS_PER_BYTE = 8
|
24
26
|
MEBI = 1024 * 1024
|
@@ -88,7 +90,7 @@ module Aspera
|
|
88
90
|
|
89
91
|
# value is provided in block
|
90
92
|
def write_file_restricted(path, force: false, mode: nil)
|
91
|
-
|
93
|
+
assert(block_given?, exception_class: Aspera::InternalError)
|
92
94
|
if force || !File.exist?(path)
|
93
95
|
# Windows may give error
|
94
96
|
File.unlink(path) rescue nil
|
@@ -2,13 +2,14 @@
|
|
2
2
|
|
3
3
|
require 'aspera/fasp/agent_base'
|
4
4
|
require 'aspera/rest'
|
5
|
+
require 'aspera/log'
|
5
6
|
require 'aspera/json_rpc'
|
6
7
|
require 'aspera/open_application'
|
7
8
|
require 'securerandom'
|
8
9
|
|
9
10
|
module Aspera
|
10
11
|
module Fasp
|
11
|
-
class
|
12
|
+
class AgentAlpha < Aspera::Fasp::AgentBase
|
12
13
|
# try twice the main init url in sequence
|
13
14
|
START_URIS = ['aspera://']
|
14
15
|
# delay between each try to start connect
|
@@ -66,53 +67,42 @@ module Aspera
|
|
66
67
|
# if there is a token, we ask connect client to use well known ssh private keys
|
67
68
|
# instead of asking password
|
68
69
|
transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
|
69
|
-
@client_app_api.start_transfer(app_id: @application_id,transfer_spec: transfer_spec)
|
70
|
-
|
70
|
+
result = @client_app_api.start_transfer(app_id: @application_id, desktop_spec: {}, transfer_spec: transfer_spec)
|
71
|
+
@xfer_id = result['uuid']
|
71
72
|
end
|
72
73
|
|
73
74
|
def wait_for_transfers_completion
|
74
|
-
client_activity_args = {'aspera_client_settings' => @client_settings}
|
75
75
|
started = false
|
76
76
|
pre_calc = false
|
77
|
-
session_id = @xfer_id
|
78
77
|
begin
|
79
78
|
loop do
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
79
|
+
transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @xfer_id)
|
80
|
+
case transfer['status']
|
81
|
+
when 'initiating', 'queued'
|
82
|
+
notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
|
83
|
+
when 'running'
|
84
|
+
if !started
|
85
|
+
notify_progress(session_id: @xfer_id, type: :session_start)
|
86
|
+
started = true
|
87
87
|
end
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
|
92
|
-
when 'running'
|
93
|
-
if !started
|
94
|
-
notify_progress(session_id: session_id, type: :session_start)
|
95
|
-
started = true
|
96
|
-
end
|
97
|
-
if !pre_calc && (transfer['bytes_expected'] != 0)
|
98
|
-
notify_progress(type: :session_size, session_id: session_id, info: transfer['bytes_expected'])
|
99
|
-
pre_calc = true
|
100
|
-
else
|
101
|
-
notify_progress(type: :transfer, session_id: session_id, info: transfer['bytes_written'])
|
102
|
-
end
|
103
|
-
when 'completed'
|
104
|
-
notify_progress(type: :end, session_id: session_id)
|
105
|
-
break
|
106
|
-
when 'failed'
|
107
|
-
notify_progress(type: :end, session_id: session_id)
|
108
|
-
raise Fasp::Error, transfer['error_desc']
|
109
|
-
when 'cancelled'
|
110
|
-
notify_progress(type: :end, session_id: session_id)
|
111
|
-
raise Fasp::Error, 'Transfer cancelled by user'
|
88
|
+
if !pre_calc && (transfer['bytes_expected'] != 0)
|
89
|
+
notify_progress(type: :session_size, session_id: @xfer_id, info: transfer['bytes_expected'])
|
90
|
+
pre_calc = true
|
112
91
|
else
|
113
|
-
notify_progress(type: :
|
114
|
-
raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
|
92
|
+
notify_progress(type: :transfer, session_id: @xfer_id, info: transfer['bytes_written'])
|
115
93
|
end
|
94
|
+
when 'completed'
|
95
|
+
notify_progress(type: :end, session_id: @xfer_id)
|
96
|
+
break
|
97
|
+
when 'failed'
|
98
|
+
notify_progress(type: :end, session_id: @xfer_id)
|
99
|
+
raise Fasp::Error, transfer['error_desc']
|
100
|
+
when 'cancelled'
|
101
|
+
notify_progress(type: :end, session_id: @xfer_id)
|
102
|
+
raise Fasp::Error, 'Transfer cancelled by user'
|
103
|
+
else
|
104
|
+
notify_progress(type: :end, session_id: @xfer_id)
|
105
|
+
raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
|
116
106
|
end
|
117
107
|
sleep(1)
|
118
108
|
end
|
@@ -121,6 +111,6 @@ module Aspera
|
|
121
111
|
end
|
122
112
|
return [:success]
|
123
113
|
end # wait
|
124
|
-
end #
|
114
|
+
end # AgentAlpha
|
125
115
|
end
|
126
116
|
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
3
5
|
module Aspera
|
4
6
|
module Fasp
|
5
7
|
# Base class for transfer agents
|
@@ -10,7 +12,9 @@ module Aspera
|
|
10
12
|
result = options.symbolize_keys
|
11
13
|
available = default.map{|k, v|"#{k}(#{v})"}.join(', ')
|
12
14
|
result.each do |k, _v|
|
13
|
-
|
15
|
+
assert_values(k, default.keys){"transfer agent parameter: #{k}"}
|
16
|
+
# check it is the expected type: too limiting, as we can have an Integer or Float, or symbol and string
|
17
|
+
# raise "Invalid value for transfer agent parameter: #{k}, expect #{default[k].class.name}" unless default[k].nil? || v.is_a?(default[k].class)
|
14
18
|
end
|
15
19
|
default.each do |k, v|
|
16
20
|
raise "Missing required agent parameter: #{k}. Parameters: #{available}" if v.eql?(:required) && !result.key?(k)
|
@@ -18,24 +22,30 @@ module Aspera
|
|
18
22
|
end
|
19
23
|
return result
|
20
24
|
end
|
21
|
-
|
25
|
+
|
26
|
+
def agent_list
|
27
|
+
Dir.entries(File.dirname(File.expand_path(__FILE__))).select do |file|
|
28
|
+
file.start_with?('agent_') && !file.eql?('agent_base.rb')
|
29
|
+
end.map{|file|file.sub(/^agent_/, '').sub(/\.rb$/, '').to_sym}
|
30
|
+
end
|
31
|
+
end
|
22
32
|
def wait_for_completion
|
23
33
|
# list of: :success or "error message string"
|
24
34
|
statuses = wait_for_transfers_completion
|
25
35
|
@progress&.reset
|
26
|
-
|
27
|
-
|
36
|
+
assert_type(statuses, Array)
|
37
|
+
assert(statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?){"bad statuses content: #{statuses}"}
|
28
38
|
return statuses
|
29
39
|
end
|
30
40
|
|
31
41
|
private
|
32
42
|
|
33
43
|
def initialize(options)
|
34
|
-
raise 'internal error' unless respond_to?(:start_transfer)
|
35
|
-
raise 'internal error' unless respond_to?(:wait_for_transfers_completion)
|
36
44
|
# method `shutdown` is optional
|
45
|
+
assert(respond_to?(:start_transfer))
|
46
|
+
assert(respond_to?(:wait_for_transfers_completion))
|
47
|
+
assert_type(options, Hash){'transfer agent options'}
|
37
48
|
Log.log.debug{Log.dump(:agent_options, options)}
|
38
|
-
raise "transfer agent options expecting Hash, but have #{options.class}" unless options.is_a?(Hash)
|
39
49
|
@progress = options[:progress]
|
40
50
|
options.delete(:progress)
|
41
51
|
end
|