aspera-cli 4.4.0 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2095 -1503
- data/bin/ascli +2 -1
- data/bin/asession +4 -5
- data/docs/test_env.conf +3 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +25 -25
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +17 -17
- data/lib/aspera/aoc.rb +238 -185
- data/lib/aspera/ascmd.rb +93 -83
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +142 -108
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +18 -21
- data/lib/aspera/cli/main.rb +173 -149
- data/lib/aspera/cli/manager.rb +163 -168
- data/lib/aspera/cli/plugin.rb +43 -31
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +405 -370
- data/lib/aspera/cli/plugins/ats.rb +86 -79
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +580 -362
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +201 -158
- data/lib/aspera/cli/plugins/faspex5.rb +80 -57
- data/lib/aspera/cli/plugins/node.rb +183 -166
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +35 -19
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +76 -113
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
- data/lib/aspera/fasp/agent_trsdk.rb +104 -0
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +152 -124
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +87 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -89
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +121 -0
- data/lib/aspera/keychain/macos_security.rb +90 -0
- data/lib/aspera/log.rb +55 -37
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -25
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +154 -135
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +116 -29
- data/docs/Makefile +0 -66
- data/docs/README.erb.md +0 -3973
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/secrets.rb +0 -20
data/bin/ascli
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
require 'rubygems'
|
|
3
|
-
$LOAD_PATH.unshift(File.dirname(__FILE__)+
|
|
4
|
+
$LOAD_PATH.unshift(File.realpath(File.dirname(File.realpath(__FILE__))+'/../lib'))
|
|
4
5
|
require 'aspera/cli/main'
|
|
5
6
|
require 'aspera/environment'
|
|
6
7
|
Encoding.default_internal = Encoding::UTF_8
|
data/bin/asession
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
# Laurent Martin/2017
|
|
3
4
|
$LOAD_PATH.unshift(File.dirname(__FILE__)+"/../lib")
|
|
4
|
-
require 'aspera/fasp/
|
|
5
|
+
require 'aspera/fasp/agent_direct'
|
|
5
6
|
require 'aspera/cli/listener/line_dump'
|
|
6
7
|
require 'aspera/cli/extended_value'
|
|
7
8
|
require 'aspera/log'
|
|
@@ -65,10 +66,8 @@ if transfer_spec.has_key?(TS_TMP_FILELIST_FOLDER)
|
|
|
65
66
|
Aspera::Fasp::Parameters.file_list_folder=transfer_spec[TS_TMP_FILELIST_FOLDER]
|
|
66
67
|
transfer_spec.delete(TS_TMP_FILELIST_FOLDER)
|
|
67
68
|
end
|
|
68
|
-
# get local agent (ascp)
|
|
69
|
-
client=Aspera::Fasp::
|
|
70
|
-
# disable ascp output on stdout to not mix with JSON events
|
|
71
|
-
client.quiet=true
|
|
69
|
+
# get local agent (ascp), disable ascp output on stdout to not mix with JSON events
|
|
70
|
+
client=Aspera::Fasp::AgentDirect.new({quiet: true})
|
|
72
71
|
# display JSON instead of legacy Lines
|
|
73
72
|
client.add_listener(Aspera::Cli::Listener::LineDump.new)
|
|
74
73
|
# start transfer (asynchronous)
|
data/docs/test_env.conf
CHANGED
|
@@ -124,6 +124,8 @@ misc:
|
|
|
124
124
|
faspex_publink_recv_from_fxuser: your value here
|
|
125
125
|
faspex_publink_send_to_fxuser: your value here
|
|
126
126
|
faspex_publink_send_to_dropbox: your value here
|
|
127
|
+
faspex_dbx: your value here
|
|
128
|
+
faspex_wkg: your value here
|
|
127
129
|
shares_upload: your value here
|
|
128
130
|
console_smart_id: your value here
|
|
129
131
|
console_smart_file: your value here
|
|
@@ -143,6 +145,7 @@ misc:
|
|
|
143
145
|
aoc_publink_folder: your value here
|
|
144
146
|
aoc_shbx_ws: your value here
|
|
145
147
|
aoc_shbx_name: your value here
|
|
148
|
+
aoc_shbx_meta: your value here
|
|
146
149
|
aoc_node1_name: your value here
|
|
147
150
|
aoc_node1_secret: your value here
|
|
148
151
|
icos_bucket_key: your value here
|
data/examples/aoc.rb
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
require 'aspera/aoc'
|
|
3
4
|
require 'aspera/log'
|
|
4
5
|
|
|
5
6
|
Aspera::Log.instance.level=:debug
|
|
6
7
|
|
|
7
|
-
if !
|
|
8
|
+
if !ARGV.length.eql?(3)
|
|
8
9
|
Aspera::Log.log.error("wrong number of args: #{ARGV.length}")
|
|
9
|
-
Aspera::Log.log.error("Usage: #{$
|
|
10
|
-
Aspera::Log.log.error("Example: #{$
|
|
10
|
+
Aspera::Log.log.error("Usage: #{$PROGRAM_NAME} <aoc URL> <aoc username> <aoc private key content>")
|
|
11
|
+
Aspera::Log.log.error("Example: #{$PROGRAM_NAME} https://myorg.ibmaspera.com john@example.com $(cat /home/john/my_key.pem)")
|
|
11
12
|
Process.exit(1)
|
|
12
13
|
end
|
|
13
14
|
|
data/examples/faspex4.rb
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
# find Faspex API here: https://developer.ibm.com/apis/catalog/?search=faspex
|
|
3
4
|
# this example makes use of class Aspera::Rest for REST calls, alternatively class RestClient of gem rest-client could be used
|
|
4
|
-
# this example makes use of class Aspera::Fasp::
|
|
5
|
+
# this example makes use of class Aspera::Fasp::AgentDirect for transfers, alternatively the official "Transfer SDK" could be used
|
|
5
6
|
# Aspera SDK can be downloaded with: `ascli conf ascp install` , it installs in $HOME/.aspera/ascli/sdk
|
|
6
7
|
require 'aspera/rest'
|
|
7
8
|
require 'aspera/log'
|
|
8
|
-
require 'aspera/fasp/
|
|
9
|
+
require 'aspera/fasp/agent_direct'
|
|
9
10
|
|
|
10
11
|
tmpdir=ENV['tmp']||Dir.tmpdir || '.'
|
|
11
12
|
|
|
@@ -17,10 +18,10 @@ Aspera::Log.instance.level=:debug
|
|
|
17
18
|
# (if data files are not there, they will be created)
|
|
18
19
|
Aspera::Fasp::Installation.instance.folder = tmpdir
|
|
19
20
|
|
|
20
|
-
if !
|
|
21
|
+
if !ARGV.length.eql?(3)
|
|
21
22
|
Aspera::Log.log.error("Wrong number of args: #{ARGV.length}")
|
|
22
|
-
Aspera::Log.log.error("Usage: #{$
|
|
23
|
-
Aspera::Log.log.error("Example: #{$
|
|
23
|
+
Aspera::Log.log.error("Usage: #{$PROGRAM_NAME} <faspex URL> <faspex username> <faspex password>")
|
|
24
|
+
Aspera::Log.log.error("Example: #{$PROGRAM_NAME} https://faspex.com/aspera/faspex john p@sSw0rd")
|
|
24
25
|
Process.exit(1)
|
|
25
26
|
end
|
|
26
27
|
|
|
@@ -36,11 +37,11 @@ Aspera::Rest.insecure=true
|
|
|
36
37
|
|
|
37
38
|
# create REST API object
|
|
38
39
|
api_v3=Aspera::Rest.new({
|
|
39
|
-
:
|
|
40
|
-
:
|
|
41
|
-
:
|
|
42
|
-
:
|
|
43
|
-
:
|
|
40
|
+
base_url: faspex_url,
|
|
41
|
+
auth: {
|
|
42
|
+
type: :basic,
|
|
43
|
+
username: faspex_user,
|
|
44
|
+
password: faspex_pass
|
|
44
45
|
}})
|
|
45
46
|
|
|
46
47
|
# very simple api call
|
|
@@ -51,7 +52,7 @@ api_v3.read('me')
|
|
|
51
52
|
|
|
52
53
|
# create a sample file to send
|
|
53
54
|
file_to_send=File.join(tmpdir,'myfile.bin')
|
|
54
|
-
File.
|
|
55
|
+
File.write(file_to_send, 'sample data')
|
|
55
56
|
# package creation parameters
|
|
56
57
|
package_create_params={'delivery'=>{'title'=>'test package','recipients'=>['aspera.user1@gmail.com'],'sources'=>[{'paths'=>[file_to_send]}]}}
|
|
57
58
|
pkg_created=api_v3.create('send',package_create_params)[:data]
|
|
@@ -59,31 +60,30 @@ pkg_created=api_v3.create('send',package_create_params)[:data]
|
|
|
59
60
|
transfer_spec=pkg_created['xfer_sessions'].first
|
|
60
61
|
# set paths of files to send
|
|
61
62
|
transfer_spec['paths']=[{'source'=>file_to_send}]
|
|
62
|
-
# get
|
|
63
|
-
transfer_client=Aspera::Fasp::
|
|
64
|
-
# disable ascp output on stdout (optional)
|
|
65
|
-
transfer_client.quiet=true
|
|
63
|
+
# get local agent (ascp), disable ascp output on stdout to not mix with JSON events
|
|
64
|
+
transfer_client=Aspera::Fasp::AgentDirect.new({quiet: true})
|
|
66
65
|
# start transfer (asynchronous)
|
|
67
66
|
job_id=transfer_client.start_transfer(transfer_spec)
|
|
67
|
+
Aspera::Log.dump('job_id',job_id)
|
|
68
68
|
# wait for all transfer completion (for the example)
|
|
69
69
|
result=transfer_client.wait_for_transfers_completion
|
|
70
70
|
# notify of any transfer error
|
|
71
|
-
result.
|
|
71
|
+
result.reject{|i|i.eql?(:success)}.each do |e|
|
|
72
72
|
Aspera::Log.log.error("A transfer error occured: #{e.message}")
|
|
73
73
|
end
|
|
74
74
|
|
|
75
75
|
# 3: Faspex 4 API v4
|
|
76
76
|
#---------------
|
|
77
77
|
api_v4=Aspera::Rest.new({
|
|
78
|
-
:
|
|
79
|
-
:
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
78
|
+
base_url: faspex_url+'/api',
|
|
79
|
+
auth: {
|
|
80
|
+
type: :oauth2,
|
|
81
|
+
base_url: faspex_url+'/auth/oauth2',
|
|
82
|
+
auth: {type: :basic, username: faspex_user, password: faspex_pass},
|
|
83
|
+
crtype: :generic,
|
|
84
|
+
generic: {grant_type: 'password'},
|
|
85
|
+
scope: 'admin'
|
|
86
|
+
}})
|
|
87
87
|
|
|
88
88
|
# Use it. Note that Faspex 4 API v4 is totally different from Faspex 4 v3 APIs, see ref on line 2
|
|
89
89
|
Aspera::Log.dump('users',api_v4.read('users')[:data])
|
data/examples/proxy.pac
CHANGED
data/examples/transfer.rb
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
# Example: transfer a file using one of the provided transfer agents
|
|
3
4
|
# location of ascp can be specified with env var "ascp"
|
|
4
5
|
# temp folder can be specified with env var "tmp"
|
|
5
|
-
require 'aspera/fasp/
|
|
6
|
+
require 'aspera/fasp/agent_direct'
|
|
6
7
|
require 'aspera/fasp/listener'
|
|
7
8
|
require 'aspera/fasp/installation'
|
|
8
9
|
require 'aspera/log'
|
|
@@ -16,7 +17,7 @@ tmpdir=ENV['tmp']||Dir.tmpdir || '.'
|
|
|
16
17
|
DEMO_CONFIG=[
|
|
17
18
|
'ssh://asperaweb@eudemo.asperademo.com:33001',
|
|
18
19
|
'https://node_asperaweb@eudemo.asperademo.com:9092',
|
|
19
|
-
'demoaspera'
|
|
20
|
+
'demoaspera'
|
|
20
21
|
]
|
|
21
22
|
|
|
22
23
|
##############################################################
|
|
@@ -26,7 +27,7 @@ DEMO_CONFIG=[
|
|
|
26
27
|
Aspera::Log.instance.level=:debug
|
|
27
28
|
|
|
28
29
|
# register aspera REST call error handlers
|
|
29
|
-
Aspera::RestErrorsAspera.
|
|
30
|
+
Aspera::RestErrorsAspera.register_handlers
|
|
30
31
|
|
|
31
32
|
# some required files are generated here (keys, certs)
|
|
32
33
|
Aspera::Fasp::Installation.instance.folder = tmpdir
|
|
@@ -36,10 +37,10 @@ Aspera::Fasp::Installation.instance.ascp_path=ENV['ascp'] if ENV.has_key?('ascp'
|
|
|
36
37
|
#Aspera::Fasp::Installation.instance.installed_products.each{|p|puts("found: #{p[:name]}")}
|
|
37
38
|
#Aspera::Fasp::Installation.instance.use_ascp_from_product('Aspera Connect')
|
|
38
39
|
# or install:
|
|
39
|
-
#
|
|
40
|
+
#
|
|
40
41
|
|
|
41
42
|
# get FASP Manager singleton based on above ascp location
|
|
42
|
-
fasp_manager=Aspera::Fasp::
|
|
43
|
+
fasp_manager=Aspera::Fasp::AgentDirect.new
|
|
43
44
|
|
|
44
45
|
# Note that it would also be possible to start transfers using other agents
|
|
45
46
|
#require 'aspera/fasp/connect'
|
|
@@ -54,7 +55,7 @@ fasp_manager=Aspera::Fasp::Local.new
|
|
|
54
55
|
class MyListener < Aspera::Fasp::Listener
|
|
55
56
|
# this is the callback called during transfers, here we only display the received information
|
|
56
57
|
# but it could be used to get detailed error information, check "type" field is "ERROR"
|
|
57
|
-
def event_enhanced(data)
|
|
58
|
+
def event_enhanced(data);$stdout.puts(JSON.generate(data));$stdout.flush;end
|
|
58
59
|
end
|
|
59
60
|
|
|
60
61
|
# register the sample listener to display events
|
|
@@ -84,9 +85,9 @@ fasp_manager.start_transfer(transfer_spec)
|
|
|
84
85
|
# get array of status, one for each session (so, a single value array)
|
|
85
86
|
# each status is either :success or "error message"
|
|
86
87
|
transfer_result=fasp_manager.wait_for_transfers_completion
|
|
87
|
-
|
|
88
|
+
$stdout.puts(JSON.generate(transfer_result))
|
|
88
89
|
# get list of errors only
|
|
89
|
-
errors=transfer_result.
|
|
90
|
+
errors=transfer_result.reject{|i|i.eql?(:success)}
|
|
90
91
|
# the transfer was not success, as there is at least one error
|
|
91
92
|
raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
|
|
92
93
|
|
|
@@ -95,19 +96,19 @@ raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
|
|
|
95
96
|
|
|
96
97
|
# create rest client for Node API on a public demo system, using public demo credentials
|
|
97
98
|
node_api=Aspera::Rest.new({
|
|
98
|
-
:
|
|
99
|
-
:
|
|
100
|
-
:
|
|
101
|
-
:
|
|
102
|
-
:
|
|
99
|
+
base_url: DEMO_CONFIG[1],
|
|
100
|
+
auth: {
|
|
101
|
+
type: :basic,
|
|
102
|
+
username: URI.parse(DEMO_CONFIG[1]).user,
|
|
103
|
+
password: DEMO_CONFIG[2]
|
|
103
104
|
}})
|
|
104
105
|
# define sample file(s) and destination folder
|
|
105
106
|
sources=["#{tmpdir}/sample_file.txt"]
|
|
106
107
|
destination='/Upload'
|
|
107
108
|
# create sample file(s)
|
|
108
|
-
sources.each{|p|File.write(p,
|
|
109
|
+
sources.each{|p|File.write(p,'Hello World!')}
|
|
109
110
|
# request transfer authorization to node for a single transfer (This is a node api v3 call)
|
|
110
|
-
send_result=node_api.create('files/upload_setup',{ :
|
|
111
|
+
send_result=node_api.create('files/upload_setup',{ transfer_requests: [{ transfer_request: { paths: [{ destination: destination }] } }] })[:data]
|
|
111
112
|
# we normally have only one transfer spec in list, so just get the first transfer_spec
|
|
112
113
|
transfer_spec=send_result['transfer_specs'].first['transfer_spec']
|
|
113
114
|
# add list of files to upload
|
|
@@ -118,7 +119,6 @@ transfer_spec['authentication']='token'
|
|
|
118
119
|
fasp_manager.start_transfer(transfer_spec)
|
|
119
120
|
# optional: wait for transfer completion helper function to get events
|
|
120
121
|
transfer_result=fasp_manager.wait_for_transfers_completion
|
|
121
|
-
errors=transfer_result.
|
|
122
|
+
errors=transfer_result.reject{|i|i.eql?(:success)}
|
|
122
123
|
# the transfer was not success, as there is at least one error
|
|
123
124
|
raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
|
|
124
|
-
|