aspera-cli 4.4.0 → 4.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2095 -1503
  3. data/bin/ascli +2 -1
  4. data/bin/asession +4 -5
  5. data/docs/test_env.conf +3 -0
  6. data/examples/aoc.rb +4 -3
  7. data/examples/faspex4.rb +25 -25
  8. data/examples/proxy.pac +1 -1
  9. data/examples/transfer.rb +17 -17
  10. data/lib/aspera/aoc.rb +238 -185
  11. data/lib/aspera/ascmd.rb +93 -83
  12. data/lib/aspera/ats_api.rb +11 -10
  13. data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
  14. data/lib/aspera/cli/extended_value.rb +42 -33
  15. data/lib/aspera/cli/formater.rb +142 -108
  16. data/lib/aspera/cli/info.rb +17 -0
  17. data/lib/aspera/cli/listener/line_dump.rb +3 -2
  18. data/lib/aspera/cli/listener/logger.rb +2 -1
  19. data/lib/aspera/cli/listener/progress.rb +16 -18
  20. data/lib/aspera/cli/listener/progress_multi.rb +18 -21
  21. data/lib/aspera/cli/main.rb +173 -149
  22. data/lib/aspera/cli/manager.rb +163 -168
  23. data/lib/aspera/cli/plugin.rb +43 -31
  24. data/lib/aspera/cli/plugins/alee.rb +6 -6
  25. data/lib/aspera/cli/plugins/aoc.rb +405 -370
  26. data/lib/aspera/cli/plugins/ats.rb +86 -79
  27. data/lib/aspera/cli/plugins/bss.rb +14 -16
  28. data/lib/aspera/cli/plugins/config.rb +580 -362
  29. data/lib/aspera/cli/plugins/console.rb +23 -19
  30. data/lib/aspera/cli/plugins/cos.rb +18 -18
  31. data/lib/aspera/cli/plugins/faspex.rb +201 -158
  32. data/lib/aspera/cli/plugins/faspex5.rb +80 -57
  33. data/lib/aspera/cli/plugins/node.rb +183 -166
  34. data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
  35. data/lib/aspera/cli/plugins/preview.rb +92 -96
  36. data/lib/aspera/cli/plugins/server.rb +79 -75
  37. data/lib/aspera/cli/plugins/shares.rb +35 -19
  38. data/lib/aspera/cli/plugins/sync.rb +20 -22
  39. data/lib/aspera/cli/transfer_agent.rb +76 -113
  40. data/lib/aspera/cli/version.rb +2 -1
  41. data/lib/aspera/colors.rb +35 -27
  42. data/lib/aspera/command_line_builder.rb +48 -34
  43. data/lib/aspera/cos_node.rb +29 -21
  44. data/lib/aspera/data_repository.rb +3 -2
  45. data/lib/aspera/environment.rb +50 -45
  46. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
  47. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
  48. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
  49. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
  50. data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
  51. data/lib/aspera/fasp/agent_trsdk.rb +104 -0
  52. data/lib/aspera/fasp/error.rb +2 -1
  53. data/lib/aspera/fasp/error_info.rb +68 -52
  54. data/lib/aspera/fasp/installation.rb +152 -124
  55. data/lib/aspera/fasp/listener.rb +1 -0
  56. data/lib/aspera/fasp/parameters.rb +87 -92
  57. data/lib/aspera/fasp/parameters.yaml +305 -249
  58. data/lib/aspera/fasp/resume_policy.rb +11 -14
  59. data/lib/aspera/fasp/transfer_spec.rb +26 -0
  60. data/lib/aspera/fasp/uri.rb +22 -21
  61. data/lib/aspera/faspex_gw.rb +55 -89
  62. data/lib/aspera/hash_ext.rb +4 -3
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +121 -0
  65. data/lib/aspera/keychain/macos_security.rb +90 -0
  66. data/lib/aspera/log.rb +55 -37
  67. data/lib/aspera/nagios.rb +13 -12
  68. data/lib/aspera/node.rb +30 -25
  69. data/lib/aspera/oauth.rb +175 -226
  70. data/lib/aspera/open_application.rb +4 -3
  71. data/lib/aspera/persistency_action_once.rb +6 -6
  72. data/lib/aspera/persistency_folder.rb +5 -9
  73. data/lib/aspera/preview/file_types.rb +6 -5
  74. data/lib/aspera/preview/generator.rb +25 -24
  75. data/lib/aspera/preview/options.rb +16 -14
  76. data/lib/aspera/preview/utils.rb +98 -98
  77. data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
  78. data/lib/aspera/proxy_auto_config.rb +111 -20
  79. data/lib/aspera/rest.rb +154 -135
  80. data/lib/aspera/rest_call_error.rb +2 -2
  81. data/lib/aspera/rest_error_analyzer.rb +23 -25
  82. data/lib/aspera/rest_errors_aspera.rb +15 -14
  83. data/lib/aspera/ssh.rb +12 -10
  84. data/lib/aspera/sync.rb +42 -41
  85. data/lib/aspera/temp_file_manager.rb +18 -14
  86. data/lib/aspera/timer_limiter.rb +2 -1
  87. data/lib/aspera/uri_reader.rb +7 -5
  88. data/lib/aspera/web_auth.rb +79 -76
  89. metadata +116 -29
  90. data/docs/Makefile +0 -66
  91. data/docs/README.erb.md +0 -3973
  92. data/docs/README.md +0 -13
  93. data/docs/diagrams.txt +0 -49
  94. data/docs/doc_tools.rb +0 -58
  95. data/lib/aspera/api_detector.rb +0 -60
  96. data/lib/aspera/cli/plugins/shares2.rb +0 -114
  97. data/lib/aspera/secrets.rb +0 -20
data/bin/ascli CHANGED
@@ -1,6 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
  require 'rubygems'
3
- $LOAD_PATH.unshift(File.dirname(__FILE__)+"/../lib")
4
+ $LOAD_PATH.unshift(File.realpath(File.dirname(File.realpath(__FILE__))+'/../lib'))
4
5
  require 'aspera/cli/main'
5
6
  require 'aspera/environment'
6
7
  Encoding.default_internal = Encoding::UTF_8
data/bin/asession CHANGED
@@ -1,7 +1,8 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
  # Laurent Martin/2017
3
4
  $LOAD_PATH.unshift(File.dirname(__FILE__)+"/../lib")
4
- require 'aspera/fasp/local'
5
+ require 'aspera/fasp/agent_direct'
5
6
  require 'aspera/cli/listener/line_dump'
6
7
  require 'aspera/cli/extended_value'
7
8
  require 'aspera/log'
@@ -65,10 +66,8 @@ if transfer_spec.has_key?(TS_TMP_FILELIST_FOLDER)
65
66
  Aspera::Fasp::Parameters.file_list_folder=transfer_spec[TS_TMP_FILELIST_FOLDER]
66
67
  transfer_spec.delete(TS_TMP_FILELIST_FOLDER)
67
68
  end
68
- # get local agent (ascp)
69
- client=Aspera::Fasp::Local.new
70
- # disable ascp output on stdout to not mix with JSON events
71
- client.quiet=true
69
+ # get local agent (ascp), disable ascp output on stdout to not mix with JSON events
70
+ client=Aspera::Fasp::AgentDirect.new({quiet: true})
72
71
  # display JSON instead of legacy Lines
73
72
  client.add_listener(Aspera::Cli::Listener::LineDump.new)
74
73
  # start transfer (asynchronous)
data/docs/test_env.conf CHANGED
@@ -124,6 +124,8 @@ misc:
124
124
  faspex_publink_recv_from_fxuser: your value here
125
125
  faspex_publink_send_to_fxuser: your value here
126
126
  faspex_publink_send_to_dropbox: your value here
127
+ faspex_dbx: your value here
128
+ faspex_wkg: your value here
127
129
  shares_upload: your value here
128
130
  console_smart_id: your value here
129
131
  console_smart_file: your value here
@@ -143,6 +145,7 @@ misc:
143
145
  aoc_publink_folder: your value here
144
146
  aoc_shbx_ws: your value here
145
147
  aoc_shbx_name: your value here
148
+ aoc_shbx_meta: your value here
146
149
  aoc_node1_name: your value here
147
150
  aoc_node1_secret: your value here
148
151
  icos_bucket_key: your value here
data/examples/aoc.rb CHANGED
@@ -1,13 +1,14 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
  require 'aspera/aoc'
3
4
  require 'aspera/log'
4
5
 
5
6
  Aspera::Log.instance.level=:debug
6
7
 
7
- if ! ARGV.length.eql?(3)
8
+ if !ARGV.length.eql?(3)
8
9
  Aspera::Log.log.error("wrong number of args: #{ARGV.length}")
9
- Aspera::Log.log.error("Usage: #{$0} <aoc URL> <aoc username> <aoc private key content>")
10
- Aspera::Log.log.error("Example: #{$0} https://myorg.ibmaspera.com john@example.com $(cat /home/john/my_key.pem)")
10
+ Aspera::Log.log.error("Usage: #{$PROGRAM_NAME} <aoc URL> <aoc username> <aoc private key content>")
11
+ Aspera::Log.log.error("Example: #{$PROGRAM_NAME} https://myorg.ibmaspera.com john@example.com $(cat /home/john/my_key.pem)")
11
12
  Process.exit(1)
12
13
  end
13
14
 
data/examples/faspex4.rb CHANGED
@@ -1,11 +1,12 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
  # find Faspex API here: https://developer.ibm.com/apis/catalog/?search=faspex
3
4
  # this example makes use of class Aspera::Rest for REST calls, alternatively class RestClient of gem rest-client could be used
4
- # this example makes use of class Aspera::Fasp::Local for transfers, alternatively the official "Transfer SDK" could be used
5
+ # this example makes use of class Aspera::Fasp::AgentDirect for transfers, alternatively the official "Transfer SDK" could be used
5
6
  # Aspera SDK can be downloaded with: `ascli conf ascp install` , it installs in $HOME/.aspera/ascli/sdk
6
7
  require 'aspera/rest'
7
8
  require 'aspera/log'
8
- require 'aspera/fasp/local'
9
+ require 'aspera/fasp/agent_direct'
9
10
 
10
11
  tmpdir=ENV['tmp']||Dir.tmpdir || '.'
11
12
 
@@ -17,10 +18,10 @@ Aspera::Log.instance.level=:debug
17
18
  # (if data files are not there, they will be created)
18
19
  Aspera::Fasp::Installation.instance.folder = tmpdir
19
20
 
20
- if ! ARGV.length.eql?(3)
21
+ if !ARGV.length.eql?(3)
21
22
  Aspera::Log.log.error("Wrong number of args: #{ARGV.length}")
22
- Aspera::Log.log.error("Usage: #{$0} <faspex URL> <faspex username> <faspex password>")
23
- Aspera::Log.log.error("Example: #{$0} https://faspex.com/aspera/faspex john p@sSw0rd")
23
+ Aspera::Log.log.error("Usage: #{$PROGRAM_NAME} <faspex URL> <faspex username> <faspex password>")
24
+ Aspera::Log.log.error("Example: #{$PROGRAM_NAME} https://faspex.com/aspera/faspex john p@sSw0rd")
24
25
  Process.exit(1)
25
26
  end
26
27
 
@@ -36,11 +37,11 @@ Aspera::Rest.insecure=true
36
37
 
37
38
  # create REST API object
38
39
  api_v3=Aspera::Rest.new({
39
- :base_url => faspex_url,
40
- :auth => {
41
- :type => :basic,
42
- :username => faspex_user,
43
- :password => faspex_pass
40
+ base_url: faspex_url,
41
+ auth: {
42
+ type: :basic,
43
+ username: faspex_user,
44
+ password: faspex_pass
44
45
  }})
45
46
 
46
47
  # very simple api call
@@ -51,7 +52,7 @@ api_v3.read('me')
51
52
 
52
53
  # create a sample file to send
53
54
  file_to_send=File.join(tmpdir,'myfile.bin')
54
- File.open(file_to_send, "w") {|f| f.write("sample data") }
55
+ File.write(file_to_send, 'sample data')
55
56
  # package creation parameters
56
57
  package_create_params={'delivery'=>{'title'=>'test package','recipients'=>['aspera.user1@gmail.com'],'sources'=>[{'paths'=>[file_to_send]}]}}
57
58
  pkg_created=api_v3.create('send',package_create_params)[:data]
@@ -59,31 +60,30 @@ pkg_created=api_v3.create('send',package_create_params)[:data]
59
60
  transfer_spec=pkg_created['xfer_sessions'].first
60
61
  # set paths of files to send
61
62
  transfer_spec['paths']=[{'source'=>file_to_send}]
62
- # get the local agent (i.e. ascp)
63
- transfer_client=Aspera::Fasp::Local.new
64
- # disable ascp output on stdout (optional)
65
- transfer_client.quiet=true
63
+ # get local agent (ascp), disable ascp output on stdout to not mix with JSON events
64
+ transfer_client=Aspera::Fasp::AgentDirect.new({quiet: true})
66
65
  # start transfer (asynchronous)
67
66
  job_id=transfer_client.start_transfer(transfer_spec)
67
+ Aspera::Log.dump('job_id',job_id)
68
68
  # wait for all transfer completion (for the example)
69
69
  result=transfer_client.wait_for_transfers_completion
70
70
  # notify of any transfer error
71
- result.select{|i|!i.eql?(:success)}.each do |e|
71
+ result.reject{|i|i.eql?(:success)}.each do |e|
72
72
  Aspera::Log.log.error("A transfer error occured: #{e.message}")
73
73
  end
74
74
 
75
75
  # 3: Faspex 4 API v4
76
76
  #---------------
77
77
  api_v4=Aspera::Rest.new({
78
- :base_url => faspex_url+'/api',
79
- :auth => {
80
- :type => :oauth2,
81
- :base_url => faspex_url+'/auth/oauth2',
82
- :grant => :header_userpass,
83
- :user_name => faspex_user,
84
- :user_pass => faspex_pass,
85
- :scope => 'admin'
86
- }})
78
+ base_url: faspex_url+'/api',
79
+ auth: {
80
+ type: :oauth2,
81
+ base_url: faspex_url+'/auth/oauth2',
82
+ auth: {type: :basic, username: faspex_user, password: faspex_pass},
83
+ crtype: :generic,
84
+ generic: {grant_type: 'password'},
85
+ scope: 'admin'
86
+ }})
87
87
 
88
88
  # Use it. Note that Faspex 4 API v4 is totally different from Faspex 4 v3 APIs, see ref on line 2
89
89
  Aspera::Log.dump('users',api_v4.read('users')[:data])
data/examples/proxy.pac CHANGED
@@ -1,4 +1,4 @@
1
- /* demo proxy pac for Amelia */
1
+ /* demo proxy pac for ascli */
2
2
  function FindProxyForURL(url, host) {
3
3
  /* Normalize the URL for pattern matching */
4
4
  url = url.toLowerCase();
data/examples/transfer.rb CHANGED
@@ -1,8 +1,9 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
  # Example: transfer a file using one of the provided transfer agents
3
4
  # location of ascp can be specified with env var "ascp"
4
5
  # temp folder can be specified with env var "tmp"
5
- require 'aspera/fasp/local'
6
+ require 'aspera/fasp/agent_direct'
6
7
  require 'aspera/fasp/listener'
7
8
  require 'aspera/fasp/installation'
8
9
  require 'aspera/log'
@@ -16,7 +17,7 @@ tmpdir=ENV['tmp']||Dir.tmpdir || '.'
16
17
  DEMO_CONFIG=[
17
18
  'ssh://asperaweb@eudemo.asperademo.com:33001',
18
19
  'https://node_asperaweb@eudemo.asperademo.com:9092',
19
- 'demoaspera',
20
+ 'demoaspera'
20
21
  ]
21
22
 
22
23
  ##############################################################
@@ -26,7 +27,7 @@ DEMO_CONFIG=[
26
27
  Aspera::Log.instance.level=:debug
27
28
 
28
29
  # register aspera REST call error handlers
29
- Aspera::RestErrorsAspera.registerHandlers
30
+ Aspera::RestErrorsAspera.register_handlers
30
31
 
31
32
  # some required files are generated here (keys, certs)
32
33
  Aspera::Fasp::Installation.instance.folder = tmpdir
@@ -36,10 +37,10 @@ Aspera::Fasp::Installation.instance.ascp_path=ENV['ascp'] if ENV.has_key?('ascp'
36
37
  #Aspera::Fasp::Installation.instance.installed_products.each{|p|puts("found: #{p[:name]}")}
37
38
  #Aspera::Fasp::Installation.instance.use_ascp_from_product('Aspera Connect')
38
39
  # or install:
39
- #
40
+ #
40
41
 
41
42
  # get FASP Manager singleton based on above ascp location
42
- fasp_manager=Aspera::Fasp::Local.new
43
+ fasp_manager=Aspera::Fasp::AgentDirect.new
43
44
 
44
45
  # Note that it would also be possible to start transfers using other agents
45
46
  #require 'aspera/fasp/connect'
@@ -54,7 +55,7 @@ fasp_manager=Aspera::Fasp::Local.new
54
55
  class MyListener < Aspera::Fasp::Listener
55
56
  # this is the callback called during transfers, here we only display the received information
56
57
  # but it could be used to get detailed error information, check "type" field is "ERROR"
57
- def event_enhanced(data);STDOUT.puts(JSON.generate(data));STDOUT.flush;end
58
+ def event_enhanced(data);$stdout.puts(JSON.generate(data));$stdout.flush;end
58
59
  end
59
60
 
60
61
  # register the sample listener to display events
@@ -84,9 +85,9 @@ fasp_manager.start_transfer(transfer_spec)
84
85
  # get array of status, one for each session (so, a single value array)
85
86
  # each status is either :success or "error message"
86
87
  transfer_result=fasp_manager.wait_for_transfers_completion
87
- STDOUT.puts(JSON.generate(transfer_result))
88
+ $stdout.puts(JSON.generate(transfer_result))
88
89
  # get list of errors only
89
- errors=transfer_result.select{|i|!i.eql?(:success)}
90
+ errors=transfer_result.reject{|i|i.eql?(:success)}
90
91
  # the transfer was not success, as there is at least one error
91
92
  raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
92
93
 
@@ -95,19 +96,19 @@ raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
95
96
 
96
97
  # create rest client for Node API on a public demo system, using public demo credentials
97
98
  node_api=Aspera::Rest.new({
98
- :base_url => DEMO_CONFIG[1],
99
- :auth => {
100
- :type => :basic,
101
- :username => URI.parse(DEMO_CONFIG[1]).user,
102
- :password => DEMO_CONFIG[2]
99
+ base_url: DEMO_CONFIG[1],
100
+ auth: {
101
+ type: :basic,
102
+ username: URI.parse(DEMO_CONFIG[1]).user,
103
+ password: DEMO_CONFIG[2]
103
104
  }})
104
105
  # define sample file(s) and destination folder
105
106
  sources=["#{tmpdir}/sample_file.txt"]
106
107
  destination='/Upload'
107
108
  # create sample file(s)
108
- sources.each{|p|File.write(p,"Hello World!")}
109
+ sources.each{|p|File.write(p,'Hello World!')}
109
110
  # request transfer authorization to node for a single transfer (This is a node api v3 call)
110
- send_result=node_api.create('files/upload_setup',{ :transfer_requests => [ { :transfer_request => { :paths => [ { :destination => destination } ] } } ] } )[:data]
111
+ send_result=node_api.create('files/upload_setup',{ transfer_requests: [{ transfer_request: { paths: [{ destination: destination }] } }] })[:data]
111
112
  # we normally have only one transfer spec in list, so just get the first transfer_spec
112
113
  transfer_spec=send_result['transfer_specs'].first['transfer_spec']
113
114
  # add list of files to upload
@@ -118,7 +119,6 @@ transfer_spec['authentication']='token'
118
119
  fasp_manager.start_transfer(transfer_spec)
119
120
  # optional: wait for transfer completion helper function to get events
120
121
  transfer_result=fasp_manager.wait_for_transfers_completion
121
- errors=transfer_result.select{|i|!i.eql?(:success)}
122
+ errors=transfer_result.reject{|i|i.eql?(:success)}
122
123
  # the transfer was not success, as there is at least one error
123
124
  raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
124
-