aspera-cli 4.24.1 → 4.25.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +1064 -745
- data/CONTRIBUTING.md +43 -100
- data/README.md +1281 -720
- data/bin/ascli +20 -1
- data/bin/asession +23 -27
- data/lib/aspera/agent/base.rb +10 -21
- data/lib/aspera/agent/connect.rb +2 -3
- data/lib/aspera/agent/desktop.rb +2 -2
- data/lib/aspera/agent/direct.rb +49 -32
- data/lib/aspera/agent/factory.rb +31 -0
- data/lib/aspera/api/aoc.rb +134 -76
- data/lib/aspera/api/cos_node.rb +3 -2
- data/lib/aspera/api/faspex.rb +213 -0
- data/lib/aspera/api/node.rb +107 -94
- data/lib/aspera/ascmd.rb +1 -2
- data/lib/aspera/ascp/installation.rb +73 -58
- data/lib/aspera/ascp/management.rb +119 -23
- data/lib/aspera/assert.rb +39 -11
- data/lib/aspera/cli/error.rb +4 -2
- data/lib/aspera/cli/extended_value.rb +91 -67
- data/lib/aspera/cli/formatter.rb +62 -27
- data/lib/aspera/cli/hints.rb +8 -0
- data/lib/aspera/cli/info.rb +4 -4
- data/lib/aspera/cli/main.rb +76 -84
- data/lib/aspera/cli/manager.rb +352 -248
- data/lib/aspera/cli/plugins/alee.rb +5 -4
- data/lib/aspera/cli/plugins/aoc.rb +175 -195
- data/lib/aspera/cli/plugins/ats.rb +4 -4
- data/lib/aspera/cli/plugins/base.rb +343 -0
- data/lib/aspera/cli/plugins/basic_auth.rb +45 -0
- data/lib/aspera/cli/plugins/config.rb +283 -269
- data/lib/aspera/cli/plugins/console.rb +27 -22
- data/lib/aspera/cli/plugins/cos.rb +3 -3
- data/lib/aspera/cli/plugins/factory.rb +78 -0
- data/lib/aspera/cli/plugins/faspex.rb +49 -46
- data/lib/aspera/cli/plugins/faspex5.rb +113 -225
- data/lib/aspera/cli/plugins/faspio.rb +19 -18
- data/lib/aspera/cli/plugins/httpgw.rb +14 -13
- data/lib/aspera/cli/plugins/node.rb +162 -149
- data/lib/aspera/cli/plugins/oauth.rb +48 -0
- data/lib/aspera/cli/plugins/orchestrator.rb +129 -45
- data/lib/aspera/cli/plugins/preview.rb +30 -50
- data/lib/aspera/cli/plugins/server.rb +21 -21
- data/lib/aspera/cli/plugins/shares.rb +45 -47
- data/lib/aspera/cli/sync_actions.rb +50 -39
- data/lib/aspera/cli/transfer_agent.rb +35 -49
- data/lib/aspera/cli/transfer_progress.rb +6 -6
- data/lib/aspera/cli/version.rb +3 -3
- data/lib/aspera/cli/wizard.rb +70 -55
- data/lib/aspera/colors.rb +6 -0
- data/lib/aspera/command_line_builder.rb +59 -61
- data/lib/aspera/command_line_converter.rb +2 -1
- data/lib/aspera/coverage.rb +2 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +51 -41
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/keychain/factory.rb +1 -2
- data/lib/aspera/keychain/macos_security.rb +1 -1
- data/lib/aspera/log.rb +37 -9
- data/lib/aspera/markdown.rb +31 -0
- data/lib/aspera/nagios.rb +7 -6
- data/lib/aspera/oauth/base.rb +25 -28
- data/lib/aspera/oauth/factory.rb +9 -9
- data/lib/aspera/oauth/url_json.rb +2 -1
- data/lib/aspera/oauth/web.rb +2 -2
- data/lib/aspera/preview/file_types.rb +23 -37
- data/lib/aspera/products/connect.rb +7 -6
- data/lib/aspera/products/desktop.rb +1 -4
- data/lib/aspera/products/other.rb +9 -1
- data/lib/aspera/products/transferd.rb +0 -1
- data/lib/aspera/rest.rb +168 -113
- data/lib/aspera/rest_error_analyzer.rb +4 -4
- data/lib/aspera/ssh.rb +7 -4
- data/lib/aspera/ssl.rb +41 -0
- data/lib/aspera/sync/args.schema.yaml +46 -3
- data/lib/aspera/sync/conf.schema.yaml +307 -123
- data/lib/aspera/sync/database.rb +2 -1
- data/lib/aspera/sync/operations.rb +135 -79
- data/lib/aspera/temp_file_manager.rb +17 -5
- data/lib/aspera/transfer/error.rb +16 -7
- data/lib/aspera/transfer/parameters.rb +35 -22
- data/lib/aspera/transfer/resumer.rb +74 -0
- data/lib/aspera/transfer/spec.rb +5 -5
- data/lib/aspera/transfer/spec.schema.yaml +170 -59
- data/lib/aspera/transfer/spec_doc.rb +49 -43
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +6 -6
- data/lib/transferd_pb.rb +2 -2
- data.tar.gz.sig +0 -0
- metadata +26 -11
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +0 -43
- data/lib/aspera/cli/plugin.rb +0 -333
- data/lib/aspera/cli/plugin_factory.rb +0 -81
- data/lib/aspera/resumer.rb +0 -77
- data/lib/aspera/transfer/error_info.rb +0 -91
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'aspera/cli/plugins/basic_auth'
|
|
3
4
|
require 'aspera/cli/plugins/node'
|
|
4
5
|
require 'aspera/assert'
|
|
5
6
|
module Aspera
|
|
6
7
|
module Cli
|
|
7
8
|
module Plugins
|
|
8
9
|
# Plugin for Aspera Shares v1
|
|
9
|
-
class Shares <
|
|
10
|
+
class Shares < BasicAuth
|
|
10
11
|
# path for node API after base url
|
|
11
12
|
NODE_API_PATH = 'node_api'
|
|
12
13
|
# path for node admin after base url
|
|
@@ -15,18 +16,13 @@ module Aspera
|
|
|
15
16
|
def detect(address_or_url)
|
|
16
17
|
address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
|
|
17
18
|
api = Rest.new(base_url: address_or_url, redirect_max: 1)
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
api.read("#{NODE_API_PATH}/app")
|
|
23
|
-
rescue RestCallError => e
|
|
24
|
-
found = true if e.response.code.to_s.eql?('401') && e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
|
|
25
|
-
end
|
|
26
|
-
return unless found
|
|
19
|
+
# TODO: use ping instead ?
|
|
20
|
+
resp = api.read("#{NODE_API_PATH}/app", exception: false, ret: :resp)
|
|
21
|
+
# shall fail: shares requires auth, but we check error message
|
|
22
|
+
return unless resp.code.to_s.eql?('401') && resp.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
|
|
27
23
|
version = 'unknown'
|
|
28
|
-
|
|
29
|
-
if (m =
|
|
24
|
+
http = api.read('login', headers: {'Accept'=>'*/*'}, ret: :resp)
|
|
25
|
+
if (m = http.body.match(/\(v(1\..*)\)/))
|
|
30
26
|
version = m[1]
|
|
31
27
|
end
|
|
32
28
|
return {
|
|
@@ -34,18 +30,20 @@ module Aspera
|
|
|
34
30
|
url: address_or_url
|
|
35
31
|
}
|
|
36
32
|
end
|
|
33
|
+
end
|
|
37
34
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
}
|
|
48
|
-
|
|
35
|
+
# @param wizard [Wizard] The wizard object
|
|
36
|
+
# @param app_url [Wizard] The wizard object
|
|
37
|
+
# @return [Hash] :preset_value, :test_args
|
|
38
|
+
def wizard(wizard, app_url)
|
|
39
|
+
return {
|
|
40
|
+
preset_value: {
|
|
41
|
+
url: app_url,
|
|
42
|
+
username: options.get_option(:username, mandatory: true),
|
|
43
|
+
password: options.get_option(:password, mandatory: true)
|
|
44
|
+
},
|
|
45
|
+
test_args: 'files browse /'
|
|
46
|
+
}
|
|
49
47
|
end
|
|
50
48
|
|
|
51
49
|
def initialize(**_)
|
|
@@ -65,19 +63,15 @@ module Aspera
|
|
|
65
63
|
when :health
|
|
66
64
|
nagios = Nagios.new
|
|
67
65
|
begin
|
|
68
|
-
|
|
66
|
+
http = Rest
|
|
69
67
|
.new(base_url: "#{options.get_option(:url, mandatory: true)}/#{NODE_API_PATH}")
|
|
70
|
-
.
|
|
71
|
-
|
|
72
|
-
subpath: 'ping',
|
|
73
|
-
headers: {'content-type': Rest::MIME_JSON}
|
|
74
|
-
)
|
|
75
|
-
raise Error, 'Shares not detected' unless res[:http].body.eql?(' ')
|
|
68
|
+
.read('ping', ret: :resp)
|
|
69
|
+
raise Error, 'Shares not detected' unless http.body.eql?(' ')
|
|
76
70
|
nagios.add_ok('shares api', 'accessible')
|
|
77
71
|
rescue StandardError => e
|
|
78
72
|
nagios.add_critical('API', e.to_s)
|
|
79
73
|
end
|
|
80
|
-
|
|
74
|
+
Main.result_object_list(nagios.status_list)
|
|
81
75
|
when :files
|
|
82
76
|
api_shares_node = basic_auth_api(NODE_API_PATH)
|
|
83
77
|
repo_command = options.get_next_command(Node::COMMANDS_SHARES)
|
|
@@ -87,21 +81,23 @@ module Aspera
|
|
|
87
81
|
when :admin
|
|
88
82
|
api_shares_admin = basic_auth_api(ADMIN_API_PATH)
|
|
89
83
|
admin_command = options.get_next_command(%i[node share transfer_settings user group].freeze)
|
|
84
|
+
lookup_share = ->(field, value){lookup_entity_generic(entity: 'share', field: field, value: value){api_shares_admin.read('data/shares')}['id']}
|
|
90
85
|
case admin_command
|
|
91
86
|
when :node
|
|
92
87
|
return entity_execute(api: api_shares_admin, entity: 'data/nodes')
|
|
93
88
|
when :share
|
|
94
|
-
share_command = options.get_next_command(%i[user_permissions group_permissions].concat(
|
|
89
|
+
share_command = options.get_next_command(%i[user_permissions group_permissions].concat(ALL_OPS))
|
|
95
90
|
case share_command
|
|
96
|
-
when *
|
|
91
|
+
when *ALL_OPS
|
|
97
92
|
return entity_execute(
|
|
98
|
-
api:
|
|
99
|
-
entity:
|
|
100
|
-
command:
|
|
101
|
-
display_fields: %w[id name node_id directory percent_free]
|
|
93
|
+
api: api_shares_admin,
|
|
94
|
+
entity: 'data/shares',
|
|
95
|
+
command: share_command,
|
|
96
|
+
display_fields: %w[id name node_id directory percent_free],
|
|
97
|
+
&lookup_share
|
|
102
98
|
)
|
|
103
99
|
when :user_permissions, :group_permissions
|
|
104
|
-
share_id = instance_identifier
|
|
100
|
+
share_id = instance_identifier(&lookup_share)
|
|
105
101
|
return entity_execute(api: api_shares_admin, entity: "data/shares/#{share_id}/#{share_command}")
|
|
106
102
|
end
|
|
107
103
|
when :transfer_settings
|
|
@@ -134,20 +130,22 @@ module Aspera
|
|
|
134
130
|
entity_commands = %i[import].freeze
|
|
135
131
|
end
|
|
136
132
|
entity_verb = options.get_next_command(entity_commands)
|
|
133
|
+
lookup_block = ->(field, value){lookup_entity_generic(entity: entity_type, field: field, value: value){api_shares_admin.read(entities_path)}['id']}
|
|
137
134
|
case entity_verb
|
|
138
|
-
when *
|
|
139
|
-
display_fields = entity_type.eql?(:user) ? %w[id username first_name last_name email] : nil
|
|
135
|
+
when *ALL_OPS # list, show, delete, create, modify
|
|
136
|
+
display_fields = entity_type.eql?(:user) ? %w[id user_id username first_name last_name email] : nil
|
|
140
137
|
display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:all)
|
|
141
138
|
return entity_execute(
|
|
142
|
-
api:
|
|
143
|
-
entity:
|
|
144
|
-
command:
|
|
145
|
-
display_fields: display_fields
|
|
139
|
+
api: api_shares_admin,
|
|
140
|
+
entity: entities_path,
|
|
141
|
+
command: entity_verb,
|
|
142
|
+
display_fields: display_fields,
|
|
143
|
+
&lookup_block
|
|
146
144
|
)
|
|
147
145
|
when *USR_GRP_SETTINGS # transfer_settings, app_authorizations, share_permissions
|
|
148
|
-
group_id = instance_identifier
|
|
146
|
+
group_id = instance_identifier(&lookup_block)
|
|
149
147
|
entities_path = "#{entities_path}/#{group_id}/#{entity_verb}"
|
|
150
|
-
return entity_execute(api: api_shares_admin, entity: entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
|
|
148
|
+
return entity_execute(api: api_shares_admin, entity: entities_path, is_singleton: !entity_verb.eql?(:share_permissions), &lookup_share)
|
|
151
149
|
when :import # saml
|
|
152
150
|
return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
|
|
153
151
|
entity_parameters = entity_parameters.transform_keys{ |k| k.gsub(/\s+/, '_').downcase}
|
|
@@ -163,7 +161,7 @@ module Aspera
|
|
|
163
161
|
api_shares_admin.create(entities_path, {entity_type=>entity_name})
|
|
164
162
|
end
|
|
165
163
|
when :users # group
|
|
166
|
-
return entity_execute(api: api_shares_admin, entity: "#{entities_path}/#{instance_identifier}/#{entities_prefix}users")
|
|
164
|
+
return entity_execute(api: api_shares_admin, entity: "#{entities_path}/#{instance_identifier(&lookup_block)}/#{entities_prefix}users")
|
|
167
165
|
else Aspera.error_unexpected_value(entity_verb)
|
|
168
166
|
end
|
|
169
167
|
end
|
|
@@ -7,9 +7,9 @@ require 'pathname'
|
|
|
7
7
|
|
|
8
8
|
module Aspera
|
|
9
9
|
module Cli
|
|
10
|
-
# Manage command line arguments to provide to Sync::
|
|
10
|
+
# Manage command line arguments to provide to Sync::Operations and Sync::Database
|
|
11
11
|
module SyncActions
|
|
12
|
-
#
|
|
12
|
+
# Translate state id (int) to string
|
|
13
13
|
STATE_STR = (['Nil'] +
|
|
14
14
|
(1..18).map{ |i| "P(#{i})"} +
|
|
15
15
|
%w[Syncd Error Confl Pconf] +
|
|
@@ -19,15 +19,19 @@ module Aspera
|
|
|
19
19
|
end
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
# Read command line arguments
|
|
23
|
-
#
|
|
22
|
+
# Read 1 or 2 command line arguments and converts to `sync_info` format
|
|
23
|
+
# The resulting sync_info has `args` format only if it contains one of the `sessions` or `instance` keys.
|
|
24
|
+
# It has the `conf` format (default) otherwise.
|
|
25
|
+
# If the `conf` format is detected, then both `local` and `remote` keys are set.
|
|
26
|
+
# @param direction [Symbol,NilClass] One of directions, or `nil` if only for admin command
|
|
24
27
|
# @return [Hash] sync info
|
|
25
28
|
def async_info_from_args(direction: nil)
|
|
26
29
|
path = options.get_next_argument('path')
|
|
27
30
|
sync_info = options.get_next_argument('sync info', mandatory: false, validation: Hash, default: {})
|
|
31
|
+
# is the positional path a remote path ?
|
|
28
32
|
path_is_remote = direction.eql?(:pull)
|
|
29
33
|
if sync_info.key?('sessions') || sync_info.key?('instance')
|
|
30
|
-
#
|
|
34
|
+
# `args`
|
|
31
35
|
sync_info['sessions'] ||= [{}]
|
|
32
36
|
Aspera.assert(sync_info['sessions'].length == 1){'Only one session is supported'}
|
|
33
37
|
session = sync_info['sessions'].first
|
|
@@ -41,7 +45,7 @@ module Aspera
|
|
|
41
45
|
local_remote = %w[local remote].map{ |i| session["#{i}_dir"]}
|
|
42
46
|
end
|
|
43
47
|
else
|
|
44
|
-
#
|
|
48
|
+
# `conf`
|
|
45
49
|
session = sync_info
|
|
46
50
|
dir_key = path_is_remote ? 'remote' : 'local'
|
|
47
51
|
session[dir_key] ||= {}
|
|
@@ -54,7 +58,7 @@ module Aspera
|
|
|
54
58
|
session[dir_key]['path'] = transfer.destination_folder(path_is_remote ? Transfer::Spec::DIRECTION_RECEIVE : Transfer::Spec::DIRECTION_SEND)
|
|
55
59
|
local_remote = %w[local remote].map{ |i| session[i]['path']}
|
|
56
60
|
end
|
|
57
|
-
#
|
|
61
|
+
# `conf` is quiet by default
|
|
58
62
|
session['quiet'] = false if !session.key?('quiet') && Environment.terminal?
|
|
59
63
|
end
|
|
60
64
|
if direction
|
|
@@ -62,17 +66,20 @@ module Aspera
|
|
|
62
66
|
session['direction'] = direction.to_s
|
|
63
67
|
# generate name if not provided by user
|
|
64
68
|
if !session.key?('name')
|
|
69
|
+
safe_char = Environment.instance.safe_filename_character
|
|
70
|
+
# from async man page:
|
|
71
|
+
# -N : can contain only ASCII alphanumeric, hyphen, and underscore characters
|
|
65
72
|
session['name'] = Environment.instance.sanitized_filename(
|
|
66
73
|
([direction.to_s] + local_remote).map do |value|
|
|
67
|
-
Pathname(value).each_filename.to_a.last(2).join(
|
|
68
|
-
end.join(
|
|
74
|
+
Pathname(value).each_filename.to_a.last(2).join(safe_char)
|
|
75
|
+
end.join(safe_char).gsub(/[^A-Za-z0-9_-]/, safe_char)
|
|
69
76
|
)
|
|
70
77
|
end
|
|
71
78
|
end
|
|
72
79
|
sync_info
|
|
73
80
|
end
|
|
74
81
|
|
|
75
|
-
#
|
|
82
|
+
# Provide database object from command line arguments for admin ops
|
|
76
83
|
def db_from_args
|
|
77
84
|
sync_info = async_info_from_args
|
|
78
85
|
session = sync_info.key?('sessions') ? sync_info['sessions'].first : sync_info
|
|
@@ -86,43 +93,47 @@ module Aspera
|
|
|
86
93
|
Sync::Database.new(Sync::Operations.session_db_file(sync_info))
|
|
87
94
|
end
|
|
88
95
|
|
|
96
|
+
def execute_sync_admin
|
|
97
|
+
command2 = options.get_next_command(%i[status find meta counters file_info overview])
|
|
98
|
+
require 'aspera/sync/database' unless command2.eql?(:status)
|
|
99
|
+
case command2
|
|
100
|
+
when :status
|
|
101
|
+
return Main.result_single_object(Sync::Operations.admin_status(async_info_from_args))
|
|
102
|
+
when :find
|
|
103
|
+
folder = options.get_next_argument('path')
|
|
104
|
+
dbs = Sync::Operations.list_db_files(folder)
|
|
105
|
+
return Main.result_object_list(dbs.keys.map{ |n| {name: n, path: dbs[n]}})
|
|
106
|
+
when :meta, :counters
|
|
107
|
+
return Main.result_single_object(db_from_args.send(command2))
|
|
108
|
+
when :file_info
|
|
109
|
+
result = db_from_args.send(command2)
|
|
110
|
+
result.each do |r|
|
|
111
|
+
r['sstate'] = SyncActions::STATE_STR[r['state']] if r['state']
|
|
112
|
+
end
|
|
113
|
+
return Main.result_object_list(
|
|
114
|
+
result,
|
|
115
|
+
fields: %w[sstate record_id f_meta_path message]
|
|
116
|
+
)
|
|
117
|
+
when :overview
|
|
118
|
+
return Main.result_object_list(
|
|
119
|
+
db_from_args.overview,
|
|
120
|
+
fields: %w[table name type]
|
|
121
|
+
)
|
|
122
|
+
else Aspera.error_unexpected_value(command2)
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
89
126
|
# Execute sync action
|
|
90
|
-
# @param &block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir
|
|
127
|
+
# @param &block [nil, Proc] block to generate transfer spec, takes: `direction` (one of DIRECTIONS), `local_dir`, `remote_dir`
|
|
91
128
|
def execute_sync_action(&block)
|
|
92
129
|
command = options.get_next_command(%i[admin] + Sync::Operations::DIRECTIONS)
|
|
93
130
|
# try to get 3 arguments as simple arguments
|
|
94
131
|
case command
|
|
95
132
|
when *Sync::Operations::DIRECTIONS
|
|
96
|
-
Sync::Operations.start(async_info_from_args(direction: command), transfer.
|
|
133
|
+
Sync::Operations.start(async_info_from_args(direction: command), transfer.user_transfer_spec, &block)
|
|
97
134
|
return Main.result_success
|
|
98
135
|
when :admin
|
|
99
|
-
|
|
100
|
-
require 'aspera/sync/database' unless command2.eql?(:status)
|
|
101
|
-
case command2
|
|
102
|
-
when :status
|
|
103
|
-
return Main.result_single_object(Sync::Operations.admin_status(async_info_from_args))
|
|
104
|
-
when :find
|
|
105
|
-
folder = options.get_next_argument('path')
|
|
106
|
-
dbs = Sync::Operations.list_db_files(folder)
|
|
107
|
-
return Main.result_object_list(dbs.keys.map{ |n| {name: n, path: dbs[n]}})
|
|
108
|
-
when :meta, :counters
|
|
109
|
-
return Main.result_single_object(db_from_args.send(command2))
|
|
110
|
-
when :file_info
|
|
111
|
-
result = db_from_args.send(command2)
|
|
112
|
-
result.each do |r|
|
|
113
|
-
r['sstate'] = SyncActions::STATE_STR[r['state']] if r['state']
|
|
114
|
-
end
|
|
115
|
-
return Main.result_object_list(
|
|
116
|
-
result,
|
|
117
|
-
fields: %w[sstate record_id f_meta_path message]
|
|
118
|
-
)
|
|
119
|
-
when :overview
|
|
120
|
-
return Main.result_object_list(
|
|
121
|
-
db_from_args.overview,
|
|
122
|
-
fields: %w[table name type]
|
|
123
|
-
)
|
|
124
|
-
else Aspera.error_unexpected_value(command2)
|
|
125
|
-
end
|
|
136
|
+
return execute_sync_admin
|
|
126
137
|
else Aspera.error_unexpected_value(command)
|
|
127
138
|
end
|
|
128
139
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'aspera/agent/
|
|
3
|
+
require 'aspera/agent/factory'
|
|
4
4
|
require 'aspera/transfer/spec'
|
|
5
5
|
require 'aspera/cli/info'
|
|
6
6
|
require 'aspera/log'
|
|
@@ -9,8 +9,8 @@ require 'aspera/assert'
|
|
|
9
9
|
module Aspera
|
|
10
10
|
module Cli
|
|
11
11
|
# The Transfer agent is a common interface to start a transfer using
|
|
12
|
-
# one of the supported transfer agents
|
|
13
|
-
#
|
|
12
|
+
# one of the supported transfer agents.
|
|
13
|
+
# Provide CLI options to select one of the transfer agents (FASP/ascp client)
|
|
14
14
|
class TransferAgent
|
|
15
15
|
# @args special value for --sources : read file list from arguments
|
|
16
16
|
FILE_LIST_FROM_ARGS = '@args'
|
|
@@ -27,12 +27,11 @@ module Aspera
|
|
|
27
27
|
<%=ts.to_yaml%>
|
|
28
28
|
END_OF_TEMPLATE
|
|
29
29
|
CP4I_REMOTE_HOST_LB = 'N/A'
|
|
30
|
-
# % (formatting bug in eclipse)
|
|
31
30
|
private_constant :FILE_LIST_FROM_ARGS,
|
|
32
31
|
:FILE_LIST_FROM_TRANSFER_SPEC,
|
|
33
32
|
:FILE_LIST_OPTIONS,
|
|
34
33
|
:DEFAULT_TRANSFER_NOTIFY_TEMPLATE
|
|
35
|
-
TRANSFER_AGENTS = Agent::
|
|
34
|
+
TRANSFER_AGENTS = Agent::Factory.instance.list.freeze
|
|
36
35
|
|
|
37
36
|
class << self
|
|
38
37
|
# @return :success if all sessions statuses returned by "start" are success
|
|
@@ -44,12 +43,13 @@ module Aspera
|
|
|
44
43
|
end
|
|
45
44
|
end
|
|
46
45
|
|
|
47
|
-
# @param
|
|
46
|
+
# @param opt_mgr [Manager] Option manager
|
|
47
|
+
# @param config_plugin [Config] Config plugin
|
|
48
48
|
def initialize(opt_mgr, config_plugin)
|
|
49
49
|
@opt_mgr = opt_mgr
|
|
50
50
|
@config = config_plugin
|
|
51
|
-
#
|
|
52
|
-
@
|
|
51
|
+
# Command line can override transfer spec
|
|
52
|
+
@user_transfer_spec = {
|
|
53
53
|
'create_dir' => true,
|
|
54
54
|
'resume_policy' => 'sparse_csum'
|
|
55
55
|
}
|
|
@@ -61,12 +61,12 @@ module Aspera
|
|
|
61
61
|
@transfer_paths = nil
|
|
62
62
|
# HTTPGW URL provided by webapp
|
|
63
63
|
@httpgw_url_lambda = nil
|
|
64
|
-
@opt_mgr.declare(:ts, 'Override transfer spec values',
|
|
64
|
+
@opt_mgr.declare(:ts, 'Override transfer spec values', allowed: Hash, handler: {o: self, m: :user_transfer_spec})
|
|
65
65
|
@opt_mgr.declare(:to_folder, 'Destination folder for transferred files')
|
|
66
66
|
@opt_mgr.declare(:sources, "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})", default: FILE_LIST_FROM_ARGS)
|
|
67
|
-
@opt_mgr.declare(:src_type, 'Type of file list',
|
|
68
|
-
@opt_mgr.declare(:transfer, 'Type of transfer agent',
|
|
69
|
-
@opt_mgr.declare(:transfer_info, 'Parameters for transfer agent',
|
|
67
|
+
@opt_mgr.declare(:src_type, 'Type of file list', allowed: %i[list pair], default: :list)
|
|
68
|
+
@opt_mgr.declare(:transfer, 'Type of transfer agent', allowed: TRANSFER_AGENTS, default: :direct)
|
|
69
|
+
@opt_mgr.declare(:transfer_info, 'Parameters for transfer agent', allowed: Hash, handler: {o: self, m: :transfer_info})
|
|
70
70
|
@opt_mgr.parse_options!
|
|
71
71
|
@notification_cb = nil
|
|
72
72
|
if !@opt_mgr.get_option(:notify_to).nil?
|
|
@@ -80,42 +80,27 @@ module Aspera
|
|
|
80
80
|
end
|
|
81
81
|
end
|
|
82
82
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
# multiple option are merged
|
|
86
|
-
def option_transfer_spec=(value)
|
|
87
|
-
Aspera.assert_type(value, Hash){'ts'}
|
|
88
|
-
@transfer_spec_command_line.deep_merge!(value)
|
|
89
|
-
end
|
|
90
|
-
|
|
91
|
-
# add other transfer spec parameters
|
|
92
|
-
def option_transfer_spec_deep_merge(value); @transfer_spec_command_line.deep_merge!(value); end
|
|
93
|
-
|
|
94
|
-
attr_reader :transfer_info
|
|
95
|
-
|
|
96
|
-
# multiple option are merged
|
|
97
|
-
def transfer_info=(value)
|
|
98
|
-
@transfer_info.deep_merge!(value)
|
|
99
|
-
end
|
|
83
|
+
attr_accessor :user_transfer_spec, :transfer_info
|
|
100
84
|
|
|
101
85
|
def agent_instance=(instance)
|
|
102
86
|
@agent = instance
|
|
103
87
|
end
|
|
104
88
|
|
|
105
89
|
# analyze options and create new agent if not already created or set
|
|
106
|
-
# TODO: make a Factory pattern
|
|
107
90
|
def agent_instance
|
|
108
91
|
return @agent unless @agent.nil?
|
|
109
92
|
agent_type = @opt_mgr.get_option(:transfer, mandatory: true)
|
|
110
93
|
# set keys as symbols
|
|
111
94
|
agent_options = @opt_mgr.get_option(:transfer_info).symbolize_keys
|
|
95
|
+
agent_options[:progress] = @config.progress_bar
|
|
96
|
+
agent_options[:config_dir] = @config.main_folder
|
|
112
97
|
# special cases
|
|
113
98
|
case agent_type
|
|
114
99
|
when :node
|
|
115
|
-
if agent_options.
|
|
100
|
+
if !agent_options.key?(:url)
|
|
116
101
|
param_set_name = @config.get_plugin_default_config_name(:node)
|
|
117
102
|
raise Cli::BadArgument, "No default node configured. Please specify #{Manager.option_name_to_line(:transfer_info)}" if param_set_name.nil?
|
|
118
|
-
agent_options
|
|
103
|
+
agent_options.merge!(@config.preset_by_name(param_set_name).symbolize_keys)
|
|
119
104
|
end
|
|
120
105
|
when :direct
|
|
121
106
|
# by default do not display ascp native progress bar
|
|
@@ -129,21 +114,20 @@ module Aspera
|
|
|
129
114
|
agent_options[:url] = @httpgw_url_lambda.call
|
|
130
115
|
end
|
|
131
116
|
end
|
|
132
|
-
agent_options[:progress] = @config.progress_bar
|
|
133
117
|
# get agent instance
|
|
134
|
-
self.agent_instance = Agent::
|
|
118
|
+
self.agent_instance = Agent::Factory.instance.create(agent_type, agent_options)
|
|
135
119
|
Log.log.debug{"transfer agent is a #{@agent.class}"}
|
|
136
120
|
return @agent
|
|
137
121
|
end
|
|
138
122
|
|
|
139
|
-
#
|
|
140
|
-
#
|
|
141
|
-
#
|
|
123
|
+
# Get destination folder
|
|
124
|
+
# @param direction [String] `send`` or `receive``
|
|
125
|
+
# @return [String] Destination folder for transfers (with default based on direction)
|
|
142
126
|
def destination_folder(direction)
|
|
143
127
|
dest_folder = @opt_mgr.get_option(:to_folder)
|
|
144
128
|
# do not expand path, if user wants to expand path: user @path:
|
|
145
129
|
return dest_folder unless dest_folder.nil?
|
|
146
|
-
dest_folder = @
|
|
130
|
+
dest_folder = @user_transfer_spec['destination_root']
|
|
147
131
|
return dest_folder unless dest_folder.nil?
|
|
148
132
|
# default: / on remote, . on local
|
|
149
133
|
case direction.to_s
|
|
@@ -161,12 +145,14 @@ module Aspera
|
|
|
161
145
|
end
|
|
162
146
|
end
|
|
163
147
|
|
|
148
|
+
# @param httpgw_url_proc [Proc]
|
|
164
149
|
def httpgw_url_cb=(httpgw_url_proc)
|
|
165
150
|
Aspera.assert_type(httpgw_url_proc, Proc){'httpgw_url_cb'}
|
|
166
151
|
@httpgw_url_lambda = httpgw_url_proc
|
|
167
152
|
end
|
|
168
153
|
|
|
169
|
-
#
|
|
154
|
+
# Transform the list of paths to a list of hash with source/dest
|
|
155
|
+
# @param file_list [Array]
|
|
170
156
|
def list_to_paths(file_list)
|
|
171
157
|
source_type = @opt_mgr.get_option(:src_type, mandatory: true)
|
|
172
158
|
case source_type
|
|
@@ -189,7 +175,7 @@ module Aspera
|
|
|
189
175
|
# return cache if set
|
|
190
176
|
return @transfer_paths unless @transfer_paths.nil?
|
|
191
177
|
# start with lower priority : get paths from transfer spec on command line
|
|
192
|
-
@transfer_paths = @
|
|
178
|
+
@transfer_paths = @user_transfer_spec['paths'] if @user_transfer_spec.key?('paths')
|
|
193
179
|
# is there a source list option ?
|
|
194
180
|
sources = @opt_mgr.get_option(:sources)
|
|
195
181
|
@transfer_paths =
|
|
@@ -212,7 +198,7 @@ module Aspera
|
|
|
212
198
|
@transfer_paths
|
|
213
199
|
when Array
|
|
214
200
|
Log.log.debug('getting file list as extended value')
|
|
215
|
-
Aspera.
|
|
201
|
+
Aspera.assert_array_all(sources, String, type: Cli::BadArgument){'sources must be a Array of String'}
|
|
216
202
|
list_to_paths(sources)
|
|
217
203
|
else Aspera.error_unexpected_value(sources){'sources'}
|
|
218
204
|
end
|
|
@@ -220,9 +206,9 @@ module Aspera
|
|
|
220
206
|
return @transfer_paths
|
|
221
207
|
end
|
|
222
208
|
|
|
223
|
-
#
|
|
209
|
+
# Start a transfer and wait for completion, plugins shall use this method
|
|
224
210
|
# @param transfer_spec [Hash]
|
|
225
|
-
# @param rest_token
|
|
211
|
+
# @param rest_token [Rest] if oauth token regeneration supported
|
|
226
212
|
def start(transfer_spec, rest_token: nil)
|
|
227
213
|
# check parameters
|
|
228
214
|
Aspera.assert_type(transfer_spec, Hash){'transfer_spec'}
|
|
@@ -231,25 +217,25 @@ module Aspera
|
|
|
231
217
|
case transfer_spec['direction']
|
|
232
218
|
when Transfer::Spec::DIRECTION_RECEIVE
|
|
233
219
|
# init default if required in any case
|
|
234
|
-
@
|
|
220
|
+
@user_transfer_spec['destination_root'] ||= destination_folder(transfer_spec['direction'])
|
|
235
221
|
when Transfer::Spec::DIRECTION_SEND
|
|
236
222
|
if transfer_spec.dig('tags', Transfer::Spec::TAG_RESERVED, 'node', 'access_key')
|
|
237
223
|
# gen4
|
|
238
|
-
@
|
|
224
|
+
@user_transfer_spec.delete('destination_root') if @user_transfer_spec.key?('destination_root_id')
|
|
239
225
|
elsif transfer_spec.key?('token')
|
|
240
226
|
# gen3
|
|
241
227
|
# in that case, destination is set in return by application (API/upload_setup)
|
|
242
228
|
# but to_folder was used in initial API call
|
|
243
|
-
@
|
|
229
|
+
@user_transfer_spec.delete('destination_root')
|
|
244
230
|
else
|
|
245
231
|
# init default if required
|
|
246
|
-
@
|
|
232
|
+
@user_transfer_spec['destination_root'] ||= destination_folder(transfer_spec['direction'])
|
|
247
233
|
end
|
|
248
234
|
end
|
|
249
235
|
# update command line paths, unless destination already has one
|
|
250
|
-
@
|
|
236
|
+
@user_transfer_spec['paths'] = transfer_spec['paths'] || ts_source_paths
|
|
251
237
|
# updated transfer spec with command line
|
|
252
|
-
transfer_spec.deep_merge!(@
|
|
238
|
+
transfer_spec.deep_merge!(@user_transfer_spec)
|
|
253
239
|
# recursively remove values that are nil (user wants to delete)
|
|
254
240
|
transfer_spec.deep_do{ |hash, key, value, _unused| hash.delete(key) if value.nil?}
|
|
255
241
|
# if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
|
|
@@ -13,7 +13,7 @@ module Aspera
|
|
|
13
13
|
class TransferProgress
|
|
14
14
|
def initialize
|
|
15
15
|
@progress_bar = nil
|
|
16
|
-
#
|
|
16
|
+
# Key is session id
|
|
17
17
|
@sessions = {}
|
|
18
18
|
@completed = false
|
|
19
19
|
@title = nil
|
|
@@ -42,21 +42,21 @@ module Aspera
|
|
|
42
42
|
progress_provided = false
|
|
43
43
|
case type
|
|
44
44
|
when :sessions_init
|
|
45
|
-
#
|
|
45
|
+
# Give opportunity to show progress of initialization with multiple status
|
|
46
46
|
Aspera.assert(session_id.nil?)
|
|
47
47
|
Aspera.assert_type(info, String)
|
|
48
|
-
#
|
|
48
|
+
# Initialization of progress bar
|
|
49
49
|
@title = info
|
|
50
50
|
when :session_start
|
|
51
51
|
Aspera.assert_type(session_id, String)
|
|
52
52
|
Aspera.assert(info.nil?)
|
|
53
53
|
raise "Session #{session_id} already started" if @sessions[session_id]
|
|
54
54
|
@sessions[session_id] = {
|
|
55
|
-
job_size: 0, #
|
|
55
|
+
job_size: 0, # Total size of transfer (pre-calc)
|
|
56
56
|
current: 0,
|
|
57
57
|
running: true
|
|
58
58
|
}
|
|
59
|
-
#
|
|
59
|
+
# Remove last pre-start message if any
|
|
60
60
|
@title = nil
|
|
61
61
|
when :session_size
|
|
62
62
|
Aspera.assert_type(session_id, String)
|
|
@@ -77,7 +77,7 @@ module Aspera
|
|
|
77
77
|
when :session_end
|
|
78
78
|
Aspera.assert_type(session_id, String)
|
|
79
79
|
Aspera.assert(info.nil?)
|
|
80
|
-
#
|
|
80
|
+
# A session may be too short and finish before it has been started
|
|
81
81
|
@sessions[session_id][:running] = false if @sessions[session_id].is_a?(Hash)
|
|
82
82
|
when :end
|
|
83
83
|
Aspera.assert(session_id.nil?)
|
data/lib/aspera/cli/version.rb
CHANGED