aspera-cli 4.21.2 → 4.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +1 -1
  4. data/CHANGELOG.md +34 -16
  5. data/CONTRIBUTING.md +6 -10
  6. data/README.md +805 -574
  7. data/examples/get_proto_file.rb +1 -1
  8. data/lib/aspera/agent/base.rb +9 -5
  9. data/lib/aspera/agent/connect.rb +30 -28
  10. data/lib/aspera/agent/desktop.rb +29 -25
  11. data/lib/aspera/agent/direct.rb +137 -125
  12. data/lib/aspera/agent/httpgw.rb +22 -26
  13. data/lib/aspera/agent/node.rb +14 -11
  14. data/lib/aspera/agent/transferd.rb +6 -2
  15. data/lib/aspera/api/aoc.rb +6 -6
  16. data/lib/aspera/api/cos_node.rb +1 -1
  17. data/lib/aspera/api/httpgw.rb +7 -3
  18. data/lib/aspera/api/node.rb +6 -4
  19. data/lib/aspera/ascmd.rb +3 -3
  20. data/lib/aspera/ascp/installation.rb +15 -16
  21. data/lib/aspera/ascp/management.rb +1 -1
  22. data/lib/aspera/assert.rb +11 -2
  23. data/lib/aspera/cli/error.rb +2 -2
  24. data/lib/aspera/cli/extended_value.rb +38 -19
  25. data/lib/aspera/cli/formatter.rb +48 -48
  26. data/lib/aspera/cli/hints.rb +1 -1
  27. data/lib/aspera/cli/main.rb +190 -168
  28. data/lib/aspera/cli/manager.rb +15 -15
  29. data/lib/aspera/cli/plugin.rb +23 -20
  30. data/lib/aspera/cli/plugin_factory.rb +1 -1
  31. data/lib/aspera/cli/plugins/alee.rb +1 -1
  32. data/lib/aspera/cli/plugins/aoc.rb +144 -107
  33. data/lib/aspera/cli/plugins/ats.rb +19 -17
  34. data/lib/aspera/cli/plugins/config.rb +67 -83
  35. data/lib/aspera/cli/plugins/console.rb +5 -3
  36. data/lib/aspera/cli/plugins/faspex.rb +39 -35
  37. data/lib/aspera/cli/plugins/faspex5.rb +104 -80
  38. data/lib/aspera/cli/plugins/faspio.rb +13 -1
  39. data/lib/aspera/cli/plugins/httpgw.rb +13 -1
  40. data/lib/aspera/cli/plugins/node.rb +306 -179
  41. data/lib/aspera/cli/plugins/orchestrator.rb +34 -40
  42. data/lib/aspera/cli/plugins/preview.rb +3 -3
  43. data/lib/aspera/cli/plugins/server.rb +6 -6
  44. data/lib/aspera/cli/plugins/shares.rb +5 -5
  45. data/lib/aspera/cli/sync_actions.rb +19 -18
  46. data/lib/aspera/cli/transfer_agent.rb +5 -5
  47. data/lib/aspera/cli/transfer_progress.rb +2 -2
  48. data/lib/aspera/cli/version.rb +1 -1
  49. data/lib/aspera/command_line_builder.rb +116 -95
  50. data/lib/aspera/coverage.rb +4 -3
  51. data/lib/aspera/environment.rb +6 -6
  52. data/lib/aspera/faspex_gw.rb +14 -14
  53. data/lib/aspera/faspex_postproc.rb +7 -6
  54. data/lib/aspera/hash_ext.rb +2 -2
  55. data/lib/aspera/json_rpc.rb +1 -1
  56. data/lib/aspera/keychain/encrypted_hash.rb +47 -34
  57. data/lib/aspera/keychain/factory.rb +41 -0
  58. data/lib/aspera/keychain/hashicorp_vault.rb +71 -0
  59. data/lib/aspera/keychain/macos_security.rb +19 -11
  60. data/lib/aspera/log.rb +28 -34
  61. data/lib/aspera/nagios.rb +6 -6
  62. data/lib/aspera/node_simulator.rb +8 -8
  63. data/lib/aspera/oauth/base.rb +8 -6
  64. data/lib/aspera/oauth/factory.rb +5 -6
  65. data/lib/aspera/oauth/url_json.rb +6 -6
  66. data/lib/aspera/persistency_action_once.rb +6 -4
  67. data/lib/aspera/persistency_folder.rb +2 -2
  68. data/lib/aspera/preview/generator.rb +1 -1
  69. data/lib/aspera/preview/options.rb +16 -16
  70. data/lib/aspera/preview/terminal.rb +3 -3
  71. data/lib/aspera/preview/utils.rb +11 -13
  72. data/lib/aspera/products/connect.rb +1 -1
  73. data/lib/aspera/products/desktop.rb +1 -1
  74. data/lib/aspera/products/transferd.rb +1 -1
  75. data/lib/aspera/proxy_auto_config.rb +2 -2
  76. data/lib/aspera/rest.rb +52 -43
  77. data/lib/aspera/rest_errors_aspera.rb +1 -1
  78. data/lib/aspera/secret_hider.rb +5 -5
  79. data/lib/aspera/ssh.rb +4 -4
  80. data/lib/aspera/transfer/convert.rb +29 -0
  81. data/lib/aspera/transfer/error_info.rb +66 -66
  82. data/lib/aspera/transfer/parameters.rb +13 -68
  83. data/lib/aspera/transfer/spec.rb +5 -6
  84. data/lib/aspera/transfer/spec.schema.yaml +753 -0
  85. data/lib/aspera/transfer/spec_doc.rb +62 -0
  86. data/lib/aspera/transfer/sync.rb +23 -72
  87. data/lib/aspera/transfer/sync_instance.schema.yaml +13 -0
  88. data/lib/aspera/transfer/sync_session.schema.yaml +79 -0
  89. data/lib/aspera/transfer/uri.rb +6 -6
  90. data/lib/aspera/uri_reader.rb +1 -1
  91. data/lib/aspera/web_auth.rb +1 -1
  92. data/lib/aspera/web_server_simple.rb +53 -44
  93. data.tar.gz.sig +1 -2
  94. metadata +37 -4
  95. metadata.gz.sig +0 -0
  96. data/examples/build_package.sh +0 -28
  97. data/lib/aspera/transfer/spec.yaml +0 -718
@@ -6,9 +6,9 @@ require 'aspera/command_line_builder'
6
6
  require 'aspera/temp_file_manager'
7
7
  require 'aspera/transfer/error'
8
8
  require 'aspera/transfer/spec'
9
+ require 'aspera/transfer/convert'
9
10
  require 'aspera/ascp/installation'
10
11
  require 'aspera/cli/formatter'
11
- require 'aspera/agent/base'
12
12
  require 'aspera/rest'
13
13
  require 'securerandom'
14
14
  require 'base64'
@@ -20,15 +20,10 @@ module Aspera
20
20
  module Transfer
21
21
  # translate transfer specification to ascp parameter list
22
22
  class Parameters
23
- # Agents shown in manual for parameters (sub list)
24
- SUPPORTED_AGENTS = Agent::Base.agent_list.freeze
25
23
  FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
26
- # Short names of columns in manual
27
- SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|agent_sym|agent_sym.to_s[0].to_sym}
24
+ private_constant :FILE_LIST_OPTIONS
28
25
  HTTP_FALLBACK_ACTIVATION_VALUES = ['1', 1, true, 'force'].freeze
29
26
 
30
- private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
31
-
32
27
  class << self
33
28
  # temp file list files are created here
34
29
  def file_list_folder=(value)
@@ -46,64 +41,14 @@ module Aspera
46
41
  @file_list_folder ||= TempFileManager.instance.new_file_path_global('asession_filelists')
47
42
  end
48
43
 
49
- # @param formatter [Cli::Formatter] formatter to use
50
- # @return a table suitable to display in manual
51
- def man_table(formatter)
52
- result = []
53
- Spec::DESCRIPTION.each do |name, options|
54
- param = {name: name, type: [options[:accepted_types]].flatten.join(','), description: options[:desc]}
55
- # add flags for supported agents in doc
56
- SUPPORTED_AGENTS.each do |agent_sym|
57
- param[agent_sym.to_s[0].to_sym] = Cli::Formatter.tick(options[:agents].nil? || options[:agents].include?(agent_sym))
58
- end
59
- # only keep lines that are usable in supported agents
60
- next if SUPPORTED_AGENTS_SHORT.inject(true){|memory, agent_short_sym|memory && param[agent_short_sym].empty?}
61
- param[:cli] =
62
- case options[:cli][:type]
63
- when :envvar then 'env:' + options[:cli][:variable]
64
- when :opt_without_arg then options[:cli][:switch]
65
- when :opt_with_arg
66
- values = if options.key?(:enum)
67
- ['enum']
68
- elsif options[:accepted_types].is_a?(Array)
69
- options[:accepted_types]
70
- elsif !options[:accepted_types].nil?
71
- [options[:accepted_types]]
72
- else
73
- raise "error: #{param}"
74
- end.map{|n|"{#{n}}"}.join('|')
75
- conversion_tag = options[:cli].key?(:convert) ? '(conversion)' : ''
76
- "#{options[:cli][:switch]} #{conversion_tag}#{values}"
77
- when :special then formatter.special_format('special')
78
- when :ignore then formatter.special_format('ignored')
79
- else
80
- param[:d].eql?(tick_yes) ? '' : 'n/a'
81
- end
82
- param[:description] += "\nAllowed values: #{options[:enum].join(', ')}" if options.key?(:enum)
83
- # replace "solidus" HTML entity with its text value
84
- param[:description] = param[:description].gsub('&sol;', '\\')
85
- result.push(param)
86
- end
87
- return result.sort_by { |parameter_info| parameter_info[:name] }
88
- end
89
-
90
- # special encoding methods used in YAML (key: :convert)
91
- def convert_remove_hyphen(value); value.tr('-', ''); end
92
-
93
- # special encoding methods used in YAML (key: :convert)
94
- def convert_json64(value); Base64.strict_encode64(JSON.generate(value)); end
95
-
96
- # special encoding methods used in YAML (key: :convert)
97
- def convert_base64(value); Base64.strict_encode64(value); end
98
-
99
44
  # file list is provided directly with ascp arguments
100
- # @param ascp_args [Array,NilClass] ascp arguments
45
+ # @columns ascp_args [Array,NilClass] ascp arguments
101
46
  def ascp_args_file_list?(ascp_args)
102
- ascp_args&.any?{|i|FILE_LIST_OPTIONS.include?(i)}
47
+ ascp_args&.any?{ |i| FILE_LIST_OPTIONS.include?(i)}
103
48
  end
104
49
  end
105
50
 
106
- # @param options [Hash] key: :wss: bool, :ascp_args: array of strings
51
+ # @columns options [Hash] key: :wss: bool, :ascp_args: array of strings
107
52
  def initialize(
108
53
  job_spec,
109
54
  ascp_args: nil,
@@ -125,7 +70,7 @@ module Aspera
125
70
  Aspera.assert(@ascp_args.all?(String)){'all ascp arguments must be String'}
126
71
  Aspera.assert_type(@trusted_certs, Array){'trusted_certs'}
127
72
  Aspera.assert_values(@client_ssh_key, Ascp::Installation::CLIENT_SSH_KEY_OPTIONS)
128
- @builder = CommandLineBuilder.new(@job_spec, Spec::DESCRIPTION)
73
+ @builder = CommandLineBuilder.new(@job_spec, Spec::SCHEMA, Convert)
129
74
  end
130
75
 
131
76
  # either place source files on command line, or add file list file
@@ -133,29 +78,29 @@ module Aspera
133
78
  # is the file list provided through ascp parameters?
134
79
  ascp_file_list_provided = self.class.ascp_args_file_list?(@ascp_args)
135
80
  # set if paths is mandatory in ts
136
- @builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided # cspell:words keepalive
81
+ @builder.required('paths', !@job_spec.key?('keepalive') && !ascp_file_list_provided) # cspell:words keepalive
137
82
  # get paths in transfer spec (after setting if it is mandatory)
138
83
  ts_paths_array = @builder.read_param('paths')
139
84
  file_list_option = nil
140
85
  # transfer spec contains paths ?
141
86
  if !ts_paths_array.nil?
142
87
  Aspera.assert(!ascp_file_list_provided){'file list provided both in transfer spec and ascp file list. Remove one of them.'}
143
- Aspera.assert(ts_paths_array.all?{|i|i.key?('source')}){"All elements of paths must have a 'source' key"}
144
- is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
145
- raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
88
+ Aspera.assert(ts_paths_array.all?{ |i| i.key?('source')}){"All elements of paths must have a 'source' key"}
89
+ is_pair_list = ts_paths_array.any?{ |i| i.key?('destination')}
90
+ raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{ |i| i.key?('destination')}
146
91
  if self.class.file_list_folder.nil?
147
92
  Aspera.assert(!is_pair_list){'file pair list is not supported when file list folder is not set'}
148
93
  # not safe for special characters ? (maybe not, depends on OS)
149
94
  Log.log.debug('placing source file list on command line (no file list file)')
150
- @builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
95
+ @builder.add_command_line_options(ts_paths_array.map{ |i| i['source']})
151
96
  else
152
97
  # safer option: generate a file list file if there is storage defined for it
153
98
  if is_pair_list
154
99
  file_list_option = '--file-pair-list'
155
- lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']) }
100
+ lines = ts_paths_array.each_with_object([]){ |e, m| m.push(e['source'], e['destination'])}
156
101
  else
157
102
  file_list_option = '--file-list'
158
- lines = ts_paths_array.map{|i|i['source']}
103
+ lines = ts_paths_array.map{ |i| i['source']}
159
104
  end
160
105
  file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
161
106
  Log.log.debug{Log.dump(:file_list, lines)}
@@ -2,7 +2,6 @@
2
2
 
3
3
  require 'aspera/command_line_builder'
4
4
  require 'aspera/assert'
5
- require 'yaml'
6
5
 
7
6
  module Aspera
8
7
  module Transfer
@@ -36,12 +35,12 @@ module Aspera
36
35
  XFER_DIR_TO_TYPE.fetch(direction)
37
36
  end
38
37
  end
39
- DESCRIPTION = CommandLineBuilder.normalize_description(YAML.load_file("#{__FILE__[0..-3]}yaml"))
38
+ SCHEMA = CommandLineBuilder.read_schema(__FILE__)
40
39
  # define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
41
- DESCRIPTION.each do |name, description|
42
- next unless description[:enum].is_a?(Array)
43
- const_set(:"#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
44
- description[:enum].each do |enum|
40
+ SCHEMA['properties'].each do |name, description|
41
+ next unless description['enum'].is_a?(Array)
42
+ const_set(:"#{name.to_s.upcase}_ENUM_VALUES", description['enum'])
43
+ description['enum'].each do |enum|
45
44
  const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
46
45
  end
47
46
  end