aspera-cli 4.4.0 → 4.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2095 -1503
  3. data/bin/ascli +2 -1
  4. data/bin/asession +4 -5
  5. data/docs/test_env.conf +3 -0
  6. data/examples/aoc.rb +4 -3
  7. data/examples/faspex4.rb +25 -25
  8. data/examples/proxy.pac +1 -1
  9. data/examples/transfer.rb +17 -17
  10. data/lib/aspera/aoc.rb +238 -185
  11. data/lib/aspera/ascmd.rb +93 -83
  12. data/lib/aspera/ats_api.rb +11 -10
  13. data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
  14. data/lib/aspera/cli/extended_value.rb +42 -33
  15. data/lib/aspera/cli/formater.rb +142 -108
  16. data/lib/aspera/cli/info.rb +17 -0
  17. data/lib/aspera/cli/listener/line_dump.rb +3 -2
  18. data/lib/aspera/cli/listener/logger.rb +2 -1
  19. data/lib/aspera/cli/listener/progress.rb +16 -18
  20. data/lib/aspera/cli/listener/progress_multi.rb +18 -21
  21. data/lib/aspera/cli/main.rb +173 -149
  22. data/lib/aspera/cli/manager.rb +163 -168
  23. data/lib/aspera/cli/plugin.rb +43 -31
  24. data/lib/aspera/cli/plugins/alee.rb +6 -6
  25. data/lib/aspera/cli/plugins/aoc.rb +405 -370
  26. data/lib/aspera/cli/plugins/ats.rb +86 -79
  27. data/lib/aspera/cli/plugins/bss.rb +14 -16
  28. data/lib/aspera/cli/plugins/config.rb +580 -362
  29. data/lib/aspera/cli/plugins/console.rb +23 -19
  30. data/lib/aspera/cli/plugins/cos.rb +18 -18
  31. data/lib/aspera/cli/plugins/faspex.rb +201 -158
  32. data/lib/aspera/cli/plugins/faspex5.rb +80 -57
  33. data/lib/aspera/cli/plugins/node.rb +183 -166
  34. data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
  35. data/lib/aspera/cli/plugins/preview.rb +92 -96
  36. data/lib/aspera/cli/plugins/server.rb +79 -75
  37. data/lib/aspera/cli/plugins/shares.rb +35 -19
  38. data/lib/aspera/cli/plugins/sync.rb +20 -22
  39. data/lib/aspera/cli/transfer_agent.rb +76 -113
  40. data/lib/aspera/cli/version.rb +2 -1
  41. data/lib/aspera/colors.rb +35 -27
  42. data/lib/aspera/command_line_builder.rb +48 -34
  43. data/lib/aspera/cos_node.rb +29 -21
  44. data/lib/aspera/data_repository.rb +3 -2
  45. data/lib/aspera/environment.rb +50 -45
  46. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
  47. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
  48. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
  49. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
  50. data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
  51. data/lib/aspera/fasp/agent_trsdk.rb +104 -0
  52. data/lib/aspera/fasp/error.rb +2 -1
  53. data/lib/aspera/fasp/error_info.rb +68 -52
  54. data/lib/aspera/fasp/installation.rb +152 -124
  55. data/lib/aspera/fasp/listener.rb +1 -0
  56. data/lib/aspera/fasp/parameters.rb +87 -92
  57. data/lib/aspera/fasp/parameters.yaml +305 -249
  58. data/lib/aspera/fasp/resume_policy.rb +11 -14
  59. data/lib/aspera/fasp/transfer_spec.rb +26 -0
  60. data/lib/aspera/fasp/uri.rb +22 -21
  61. data/lib/aspera/faspex_gw.rb +55 -89
  62. data/lib/aspera/hash_ext.rb +4 -3
  63. data/lib/aspera/id_generator.rb +8 -7
  64. data/lib/aspera/keychain/encrypted_hash.rb +121 -0
  65. data/lib/aspera/keychain/macos_security.rb +90 -0
  66. data/lib/aspera/log.rb +55 -37
  67. data/lib/aspera/nagios.rb +13 -12
  68. data/lib/aspera/node.rb +30 -25
  69. data/lib/aspera/oauth.rb +175 -226
  70. data/lib/aspera/open_application.rb +4 -3
  71. data/lib/aspera/persistency_action_once.rb +6 -6
  72. data/lib/aspera/persistency_folder.rb +5 -9
  73. data/lib/aspera/preview/file_types.rb +6 -5
  74. data/lib/aspera/preview/generator.rb +25 -24
  75. data/lib/aspera/preview/options.rb +16 -14
  76. data/lib/aspera/preview/utils.rb +98 -98
  77. data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
  78. data/lib/aspera/proxy_auto_config.rb +111 -20
  79. data/lib/aspera/rest.rb +154 -135
  80. data/lib/aspera/rest_call_error.rb +2 -2
  81. data/lib/aspera/rest_error_analyzer.rb +23 -25
  82. data/lib/aspera/rest_errors_aspera.rb +15 -14
  83. data/lib/aspera/ssh.rb +12 -10
  84. data/lib/aspera/sync.rb +42 -41
  85. data/lib/aspera/temp_file_manager.rb +18 -14
  86. data/lib/aspera/timer_limiter.rb +2 -1
  87. data/lib/aspera/uri_reader.rb +7 -5
  88. data/lib/aspera/web_auth.rb +79 -76
  89. metadata +116 -29
  90. data/docs/Makefile +0 -66
  91. data/docs/README.erb.md +0 -3973
  92. data/docs/README.md +0 -13
  93. data/docs/diagrams.txt +0 -49
  94. data/docs/doc_tools.rb +0 -58
  95. data/lib/aspera/api_detector.rb +0 -60
  96. data/lib/aspera/cli/plugins/shares2.rb +0 -114
  97. data/lib/aspera/secrets.rb +0 -20
data/docs/README.md DELETED
@@ -1,13 +0,0 @@
1
- docs/README.erb.md uses markdown format with embeded ruby macros (erb)
2
-
3
- /README.md is generated after compilation of README.erb.md
4
-
5
- docs/README.erb.md contains the following macros:
6
-
7
- * `<%=cmd%>` just the command line tool name
8
- * `<%=tool%>` the tool name in courrier to be included in text paragraphs
9
- * `<%=evp%>` env var prefix
10
- * `<%=opprst%>` option preset
11
- * `<%=prst%>` link to preset section, name and link to preset
12
- * `<%=prstt%>` preset in title
13
-
data/docs/diagrams.txt DELETED
@@ -1,49 +0,0 @@
1
-
2
- The following section contains the diagrams posted on support KB.
3
- mde with: https://www.websequencediagrams.com/
4
-
5
-
6
- -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
7
- title general Aspera Files APIs use
8
- Client->+Files API: GET authorization(parameters)
9
- Files API-->-Client: Result=Bearer token
10
- Client-->Files API: use APIs with Authorization=Bearer
11
-
12
-
13
- -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
14
- title basic OAuth with grant_type=password (username/password, only local users, not google, SAML)
15
- Client->+Files API: GET authorization(username/password)
16
- Files API-->-Client: Result=Bearer token
17
- Client-->Files API: use APIs with Authorization=Bearer
18
-
19
-
20
- -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
21
- title OAuth with grant_type=code (any user: local users, google, SAML)
22
- note right of Client
23
- * Client is registered in "Files", with redirect_uri
24
- * Files generates client_id and secret
25
- * Listen on "redirect_uri" end point (TCP port)
26
- * build "login address" for user:
27
- api.asperafiles.com/oauth2/<org>/authorize
28
- * open a browser for user with this URL
29
- end note
30
- Client->*Local Port: listen
31
- Client->Browser: open login url in browser (give redirect_uri and scope)
32
- Browser->Files: load login page
33
- note right of Browser: user clicks on "google auth"
34
- Browser->+Google: display google login page
35
- note right of Browser: user authenticates
36
- Google-->-Browser: browser is redirected to the "redirect URI" providing a "code"
37
- Browser->Local Port: browser provides code to client
38
- Local Port-->Client: read code
39
- Client->Local Port: close
40
- destroy Local Port
41
- note right of Client: close listening port after use
42
- Client->+Files: GET oauth/token(grant_type=code)
43
- Files-->-Client: Bearer token
44
- Client-->Files API: use APIs with Authorization=Bearer
45
-
46
-
47
-
48
-
49
-
data/docs/doc_tools.rb DELETED
@@ -1,58 +0,0 @@
1
- # get transfer spec parameter description
2
- $LOAD_PATH.unshift(ENV["INCL_DIR_GEM"])
3
- require 'aspera/fasp/parameters'
4
-
5
- # check that required env vars exist, and files
6
- %w{EXENAME GEMSPEC INCL_USAGE INCL_COMMANDS INCL_ASESSION INCL_DIR_GEM}.each do |e|
7
- raise "missing env var #{e}" unless ENV.has_key?(e)
8
- raise "missing file #{ENV[e]}" unless File.exist?(ENV[e]) or !e.start_with?('INCL_')
9
- end
10
-
11
- # set values used in ERB
12
- # just command name
13
- def cmd;ENV['EXENAME'];end
14
-
15
- # used in text with formatting of command
16
- def tool;'`'+cmd+'`';end
17
-
18
- # prefix for env vars
19
- def evp;cmd.upcase+'_';end
20
-
21
- # just the name for "option preset"
22
- def opprst;'option preset';end
23
-
24
- # name with link
25
- def prst;'['+opprst+'](#lprt)';end
26
-
27
- # name with link (plural)
28
- def prsts;'['+opprst+'s](#lprt)';end
29
-
30
- # in title
31
- def prstt;opprst.capitalize;end
32
-
33
- def gemspec;Gem::Specification::load(ENV['GEMSPEC']) or raise "error loading #{ENV["GEMSPEC"]}";end
34
-
35
- def geminstadd;gemspec.version.to_s.match(/\.[^0-9]/)?' --pre':'';end
36
-
37
- # transfer spec description generation
38
- def spec_table
39
- r='<table><tr><th>Field</th><th>Type</th>'
40
- Aspera::Fasp::Parameters::SUPPORTED_AGENTS_SHORT.each do |c|
41
- r << '<th>'<<c.to_s.upcase<<'</th>'
42
- end
43
- r << '<th>Description</th></tr>'
44
- Aspera::Fasp::Parameters.man_table.each do |p|
45
- p[:description] << (p[:description].empty? ? '' : "\n") << "(" << p[:cli] << ")" unless p[:cli].to_s.empty?
46
- p.delete(:cli)
47
- p.keys.each{|c|p[c]='&nbsp;' if p[c].to_s.empty?}
48
- p[:description].gsub!("\n",'<br/>')
49
- p[:type].gsub!(',','<br/>')
50
- r << '<tr><td>'<<p[:name]<<'</td><td>'<<p[:type]<<'</td>'
51
- Aspera::Fasp::Parameters::SUPPORTED_AGENTS_SHORT.each do |c|
52
- r << '<td>'<<p[c]<<'</td>'
53
- end
54
- r << '<td>'<<p[:description]<<'</td></tr>'
55
- end
56
- r << '</table>'
57
- return r
58
- end
@@ -1,60 +0,0 @@
1
- require 'aspera/log'
2
- require 'aspera/rest'
3
-
4
- module Aspera
5
- # detect Aspera product by calling API
6
- class ApiDetector
7
- # @return a hash: {:product=>:node,:version=>'unknown'}
8
- # if not found: {:product=>:unknown,:version=>'unknown'}
9
- def self.discover_product(url)
10
- #uri=URI.parse(url)
11
- api=Rest.new({:base_url=>url})
12
- # Node
13
- begin
14
- result=api.call({:operation=>'GET',:subpath=>'ping'})
15
- if result[:http].body.eql?('')
16
- return {:product=>:node,:version=>'unknown'}
17
- end
18
- rescue SocketError => e
19
- raise e
20
- rescue => e
21
- Log.log.debug("not node (#{e.class}: #{e})")
22
- end
23
- # AoC
24
- begin
25
- result=api.call({:operation=>'GET',:subpath=>'',:headers=>{'Accept'=>'text/html'}})
26
- if result[:http].body.include?('content="AoC"')
27
- return {:product=>:aoc,:version=>'unknown'}
28
- end
29
- rescue SocketError => e
30
- raise e
31
- rescue => e
32
- Log.log.debug("not aoc (#{e.class}: #{e})")
33
- end
34
- # Faspex
35
- begin
36
- result=api.call({:operation=>'POST',:subpath=>'aspera/faspex',:headers=>{'Accept'=>'application/xrds+xml'},:text_body_params=>''})
37
- if result[:http].body.start_with?('<?xml')
38
- res_s=XmlSimple.xml_in(result[:http].body, {"ForceArray" => false})
39
- version=res_s['XRD']['application']['version']
40
- #return JSON.pretty_generate(res_s)
41
- end
42
- return {:product=>:faspex,:version=>version}
43
- rescue
44
- Log.log.debug("not faspex")
45
- end
46
- # Shares
47
- begin
48
- result=api.read('node_api/app')
49
- Log.log.warn("not supposed to work")
50
- rescue RestCallError => e
51
- if e.response.code.to_s.eql?('401') and e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
52
- return {:product=>:shares,:version=>'unknown'}
53
- end
54
- Log.log.warn("not shares: #{e.response.code} #{e.response.body}")
55
- rescue
56
- end
57
- return {:product=>:unknown,:version=>'unknown'}
58
- end
59
- end
60
- end
@@ -1,114 +0,0 @@
1
- require 'aspera/cli/plugins/node'
2
- require 'uri'
3
-
4
- module Aspera
5
- module Cli
6
- module Plugins
7
- class Shares2 < BasicAuthPlugin
8
- def initialize(env)
9
- super(env)
10
- self.options.add_opt_simple(:organization,"organization")
11
- self.options.add_opt_simple(:project,"project")
12
- self.options.add_opt_simple(:share,"share")
13
- self.options.parse_options!
14
- return if env[:man_only]
15
- unless env[:man_only]
16
- # get parameters
17
- shares2_api_base_url=self.options.get_option(:url,:mandatory)
18
- shares2_username=self.options.get_option(:username,:mandatory)
19
- shares2_password=self.options.get_option(:password,:mandatory)
20
-
21
- # create object for REST calls to Shares2
22
- @api_shares2_oauth=Rest.new({
23
- :base_url => shares2_api_base_url,
24
- :auth => {
25
- :type => :oauth2,
26
- :base_url => shares2_api_base_url+'/oauth2',
27
- :grant => :header_userpass,
28
- :user_name => shares2_username,
29
- :user_pass => shares2_password
30
- }})
31
-
32
- @api_node=Rest.new({
33
- :base_url => shares2_api_base_url+'/node_api',
34
- :auth => {
35
- :type => :basic,
36
- :username => shares2_username,
37
- :password => shares2_password}})
38
- end
39
- end
40
-
41
- # path_prefix is either "" or "res/id/"
42
- # adds : prefix+"res/id/"
43
- # modify parameter string
44
- def set_resource_path_by_id_or_name(resource_path,resource_sym)
45
- res_id=self.options.get_option(resource_sym,:mandatory)
46
- # lets get the class path
47
- resource_path<<resource_sym.to_s+'s'
48
- # is this an integer ? or a name
49
- if res_id.to_i.to_s != res_id
50
- all=@api_shares2_oauth.read(resource_path)[:data]
51
- one=all.select{|i|i['name'].start_with?(res_id)}
52
- Log.log.debug(one)
53
- raise CliBadArgument,"No matching name for #{res_id} in #{all}" if one.empty?
54
- raise CliBadArgument,"More than one match: #{one}" if one.length > 1
55
- res_id=one.first['id'].to_s
56
- end
57
- Log.log.debug("res_id=#{res_id}")
58
- resource_path<<'/'+res_id+'/'
59
- return resource_path
60
- end
61
-
62
- # path_prefix is empty or ends with slash
63
- def process_entity_action(resource_sym,path_prefix)
64
- resource_path=path_prefix+resource_sym.to_s+'s'
65
- operations=[:list,:create,:delete]
66
- command=self.options.get_next_command(operations)
67
- case command
68
- when :create
69
- params=self.options.get_next_argument("creation data (json structure)")
70
- resp=@api_shares2_oauth.create(resource_path,params)
71
- return {:data=>resp[:data],:type => :other_struct}
72
- when :list
73
- default_fields=['id','name']
74
- query=self.options.get_option(:query,:optional)
75
- args=query.nil? ? nil : {'json_query'=>query}
76
- Log.log.debug("#{args}".bg_red)
77
- return {:data=>@api_shares2_oauth.read(resource_path,args)[:data],:fields=>default_fields,:type=>:object_list}
78
- when :delete
79
- @api_shares2_oauth.delete(set_resource_path_by_id_or_name(path_prefix,resource_sym))
80
- return Main.result_status('deleted')
81
- when :info
82
- return {:type=>:other_struct,:data=>@api_shares2_oauth.read(set_resource_path_by_id_or_name(path_prefix,resource_sym),args)[:data]}
83
- else raise :ERROR
84
- end
85
- end
86
-
87
- ACTIONS=[ :repository,:organization,:project,:team,:share,:appinfo,:userinfo,:admin]
88
-
89
- def execute_action
90
- command=self.options.get_next_command(ACTIONS)
91
- case command
92
- when :repository
93
- command=self.options.get_next_command(Node::COMMON_ACTIONS)
94
- return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: @api_node)).execute_action(command)
95
- when :appinfo
96
- node_info=@api_node.call({:operation=>'GET',:subpath=>'app',:headers=>{'Accept'=>'application/json','Content-Type'=>'application/json'}})[:data]
97
- return { :type=>:single_object ,:data => node_info }
98
- when :userinfo
99
- node_info=@api_node.call({:operation=>'GET',:subpath=>'current_user',:headers=>{'Accept'=>'application/json','Content-Type'=>'application/json'}})[:data]
100
- return { :type=>:single_object ,:data => node_info }
101
- when :organization,:project,:share,:team
102
- prefix=''
103
- set_resource_path_by_id_or_name(prefix,:organization) if [:project,:team,:share].include?(command)
104
- set_resource_path_by_id_or_name(prefix,:project) if [:share].include?(command)
105
- process_entity_action(command,prefix)
106
- when :admin
107
- command=self.options.get_next_command([:users,:groups,:nodes])
108
- return self.entity_action(@api_shares2_oauth,"system/#{command}",nil,:id)
109
- end # command
110
- end # execute_action
111
- end # Files
112
- end # Plugins
113
- end # Cli
114
- end # Aspera
@@ -1,20 +0,0 @@
1
- module Aspera
2
- # Manage secrets in CLI using secure way (encryption, wallet, etc...)
3
- class Secrets
4
- attr_accessor :default_secret,:all_secrets
5
- def initialize()
6
- @default_secret=nil
7
- @all_secrets={}
8
- end
9
-
10
- def get_secret(id=nil,mandatory=true)
11
- secret=@default_secret || @all_secrets[id]
12
- raise "please provide secret for #{id}" if secret.nil? and mandatory
13
- return secret
14
- end
15
-
16
- def get_secrets
17
- return @all_secrets
18
- end
19
- end
20
- end