aspera-cli 4.28.0 → 4.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +143 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +103 -26
  10. data/SECURITY.md +70 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1633 -609
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +20 -20
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +71 -65
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +354 -0
  41. data/lib/aspera/ascp/management.rb +16 -8
  42. data/lib/aspera/ascp/session_errors.rb +217 -0
  43. data/lib/aspera/assert.rb +4 -0
  44. data/lib/aspera/cli/ascp_actions.rb +12 -19
  45. data/lib/aspera/cli/ascp_logs_actions.rb +176 -0
  46. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  47. data/lib/aspera/cli/bootstrapper.rb +37 -33
  48. data/lib/aspera/cli/command_line.rb +24 -6
  49. data/lib/aspera/cli/command_registry.rb +6 -1
  50. data/lib/aspera/cli/command_spec.rb +34 -7
  51. data/lib/aspera/cli/completion/ascli.bash +19 -5
  52. data/lib/aspera/cli/completion/ascli.fish +13 -2
  53. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  54. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  55. data/lib/aspera/cli/completion_actions.rb +303 -69
  56. data/lib/aspera/cli/context.rb +9 -4
  57. data/lib/aspera/cli/deprecation.rb +5 -2
  58. data/lib/aspera/cli/error.rb +3 -0
  59. data/lib/aspera/cli/extended_value.rb +61 -27
  60. data/lib/aspera/cli/formatter.rb +29 -22
  61. data/lib/aspera/cli/gem_checker.rb +9 -6
  62. data/lib/aspera/cli/hints.rb +22 -10
  63. data/lib/aspera/cli/http.rb +12 -9
  64. data/lib/aspera/cli/info.rb +12 -7
  65. data/lib/aspera/cli/mailer.rb +6 -3
  66. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  67. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  68. data/lib/aspera/cli/mcp/tool.rb +15 -14
  69. data/lib/aspera/cli/option_declarator.rb +10 -4
  70. data/lib/aspera/cli/option_registry.rb +6 -3
  71. data/lib/aspera/cli/option_types.rb +17 -2
  72. data/lib/aspera/cli/option_value.rb +65 -12
  73. data/lib/aspera/cli/options.schema.yaml +487 -63
  74. data/lib/aspera/cli/parser.rb +56 -32
  75. data/lib/aspera/cli/plugins/alee.rb +7 -4
  76. data/lib/aspera/cli/plugins/aoc.rb +244 -35
  77. data/lib/aspera/cli/plugins/ats.rb +9 -8
  78. data/lib/aspera/cli/plugins/base.rb +17 -12
  79. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  80. data/lib/aspera/cli/plugins/config.rb +198 -103
  81. data/lib/aspera/cli/plugins/console.rb +6 -3
  82. data/lib/aspera/cli/plugins/cos.rb +8 -4
  83. data/lib/aspera/cli/plugins/factory.rb +21 -7
  84. data/lib/aspera/cli/plugins/faspex5.rb +28 -25
  85. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  86. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  87. data/lib/aspera/cli/plugins/mcp.rb +28 -22
  88. data/lib/aspera/cli/plugins/node.rb +31 -30
  89. data/lib/aspera/cli/plugins/oauth.rb +5 -1
  90. data/lib/aspera/cli/plugins/orchestrator.rb +272 -72
  91. data/lib/aspera/cli/plugins/preview.rb +69 -44
  92. data/lib/aspera/cli/plugins/server.rb +63 -11
  93. data/lib/aspera/cli/plugins/shares.rb +86 -16
  94. data/lib/aspera/cli/plugins/transferd.rb +223 -0
  95. data/lib/aspera/cli/preset_actions.rb +144 -30
  96. data/lib/aspera/cli/preset_manager.rb +52 -18
  97. data/lib/aspera/cli/prompt.rb +4 -1
  98. data/lib/aspera/cli/result.rb +12 -8
  99. data/lib/aspera/cli/runner.rb +69 -46
  100. data/lib/aspera/cli/secret_finder.rb +4 -1
  101. data/lib/aspera/cli/security.rb +79 -9
  102. data/lib/aspera/cli/special_values.rb +4 -1
  103. data/lib/aspera/cli/sync_actions.rb +13 -10
  104. data/lib/aspera/cli/table_fitter.rb +3 -0
  105. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  106. data/lib/aspera/cli/transfer_actions.rb +11 -8
  107. data/lib/aspera/cli/transfer_agent.rb +29 -20
  108. data/lib/aspera/cli/transfer_progress.rb +13 -45
  109. data/lib/aspera/cli/vault_actions.rb +45 -0
  110. data/lib/aspera/cli/vault_manager.rb +188 -42
  111. data/lib/aspera/cli/version.rb +4 -1
  112. data/lib/aspera/cli/web_secret.rb +12 -10
  113. data/lib/aspera/cli/wizard.rb +42 -39
  114. data/lib/aspera/command_line_builder.rb +9 -5
  115. data/lib/aspera/command_line_converter.rb +4 -1
  116. data/lib/aspera/coverage.rb +103 -33
  117. data/lib/aspera/data_repository.rb +4 -1
  118. data/lib/aspera/dot_container.rb +4 -1
  119. data/lib/aspera/duration.rb +5 -2
  120. data/lib/aspera/environment.rb +21 -12
  121. data/lib/aspera/exec_spec.rb +3 -0
  122. data/lib/aspera/faspex_gw.rb +13 -9
  123. data/lib/aspera/faspex_postproc.rb +15 -12
  124. data/lib/aspera/formatter_interface.rb +4 -1
  125. data/lib/aspera/graphql.rb +5 -1
  126. data/lib/aspera/hash_ext.rb +3 -0
  127. data/lib/aspera/id_generator.rb +4 -2
  128. data/lib/aspera/json_rpc/client.rb +6 -5
  129. data/lib/aspera/json_rpc/version.rb +3 -0
  130. data/lib/aspera/line_logger.rb +4 -1
  131. data/lib/aspera/link_header.rb +4 -1
  132. data/lib/aspera/log.rb +18 -5
  133. data/lib/aspera/markdown.rb +5 -2
  134. data/lib/aspera/mime.rb +3 -0
  135. data/lib/aspera/nagios.rb +6 -4
  136. data/lib/aspera/node_emulator.rb +15 -13
  137. data/lib/aspera/oauth/client/base.rb +91 -70
  138. data/lib/aspera/oauth/client/boot.rb +7 -4
  139. data/lib/aspera/oauth/client/factory.rb +6 -3
  140. data/lib/aspera/oauth/client/generic.rb +5 -1
  141. data/lib/aspera/oauth/client/json_credentials.rb +4 -1
  142. data/lib/aspera/oauth/client/jwt.rb +6 -3
  143. data/lib/aspera/oauth/client/shares_web.rb +85 -0
  144. data/lib/aspera/oauth/client/url_json.rb +4 -1
  145. data/lib/aspera/oauth/client/web.rb +39 -12
  146. data/lib/aspera/oauth/server/pkce.rb +6 -5
  147. data/lib/aspera/orchestrator_workflow_checker.rb +119 -1
  148. data/lib/aspera/persistency_action_once.rb +5 -3
  149. data/lib/aspera/persistency_folder.rb +6 -3
  150. data/lib/aspera/preview/file_types.rb +6 -3
  151. data/lib/aspera/preview/generator.rb +9 -5
  152. data/lib/aspera/preview/options.schema.yaml +3 -3
  153. data/lib/aspera/preview/terminal.rb +8 -4
  154. data/lib/aspera/preview/utils.rb +6 -4
  155. data/lib/aspera/products/connect.rb +5 -2
  156. data/lib/aspera/products/desktop.rb +4 -1
  157. data/lib/aspera/products/other.rb +4 -2
  158. data/lib/aspera/products/transferd.rb +6 -2
  159. data/lib/aspera/products/transferd_service.rb +610 -0
  160. data/lib/aspera/proxy_auto_config.js +2 -2
  161. data/lib/aspera/proxy_auto_config.rb +5 -2
  162. data/lib/aspera/rainbow.rb +3 -0
  163. data/lib/aspera/rest/aspera_errors.rb +4 -1
  164. data/lib/aspera/rest/call_error.rb +3 -0
  165. data/lib/aspera/rest/client.rb +22 -29
  166. data/lib/aspera/rest/error_analyzer.rb +5 -2
  167. data/lib/aspera/rest/list.rb +8 -4
  168. data/lib/aspera/rest/parameters.rb +3 -0
  169. data/lib/aspera/rest/util.rb +8 -5
  170. data/lib/aspera/rest.rb +9 -6
  171. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  172. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  173. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  174. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  175. data/lib/aspera/schema/async_tables.yaml +11 -10
  176. data/lib/aspera/schema/documentation.rb +8 -2
  177. data/lib/aspera/schema/reader.rb +7 -0
  178. data/lib/aspera/schema/registry.rb +43 -28
  179. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  180. data/lib/aspera/schema/validator.rb +4 -1
  181. data/lib/aspera/sealed_hash.rb +166 -0
  182. data/lib/aspera/secret/hider.rb +201 -0
  183. data/lib/aspera/secret/reference.rb +150 -0
  184. data/lib/aspera/secret/resolver.rb +212 -0
  185. data/lib/aspera/secret/store/base.rb +153 -0
  186. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  187. data/lib/aspera/secret/store/factory.rb +103 -0
  188. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  189. data/lib/aspera/secret/store/keeper.rb +141 -0
  190. data/lib/aspera/secret/store/macos_security.rb +284 -0
  191. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  192. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  193. data/lib/aspera/secret/store/one_password_cli.rb +110 -0
  194. data/lib/aspera/secret/store/secret_service.rb +125 -0
  195. data/lib/aspera/secret/strength.rb +119 -0
  196. data/lib/aspera/servlet_helpers.rb +5 -2
  197. data/lib/aspera/ssh.rb +10 -7
  198. data/lib/aspera/ssl.rb +5 -2
  199. data/lib/aspera/string_ext.rb +3 -0
  200. data/lib/aspera/sync/args.schema.yaml +3 -0
  201. data/lib/aspera/sync/conf.schema.yaml +9 -4
  202. data/lib/aspera/sync/database.rb +3 -0
  203. data/lib/aspera/sync/operations.rb +16 -14
  204. data/lib/aspera/temp_file_manager.rb +8 -4
  205. data/lib/aspera/timer_limiter.rb +3 -0
  206. data/lib/aspera/transfer/error.rb +4 -1
  207. data/lib/aspera/transfer/faux_file.rb +4 -1
  208. data/lib/aspera/transfer/parameters.rb +17 -14
  209. data/lib/aspera/transfer/result.rb +3 -0
  210. data/lib/aspera/transfer/resumer.rb +6 -4
  211. data/lib/aspera/transfer/spec.rb +9 -22
  212. data/lib/aspera/transfer/units.rb +75 -0
  213. data/lib/aspera/uri_reader.rb +57 -7
  214. data/lib/aspera/web_auth/index.html.erb +6 -2
  215. data/lib/aspera/web_auth/paste.rb +109 -0
  216. data/lib/aspera/web_auth/receiver.rb +50 -0
  217. data/lib/aspera/web_auth/relay.rb +76 -0
  218. data/lib/aspera/web_auth/reply.html.erb +150 -0
  219. data/lib/aspera/web_auth/server.rb +47 -10
  220. data/lib/aspera/web_page.rb +7 -2
  221. data/lib/aspera/web_server_simple.rb +6 -3
  222. data/lib/aspera/yaml.rb +5 -1
  223. data/llms.txt +81 -0
  224. data/skills/ascli/SKILL.md +88 -0
  225. data.tar.gz.sig +0 -0
  226. metadata +72 -19
  227. metadata.gz.sig +0 -0
  228. data/lib/aspera/keychain/base.rb +0 -30
  229. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  230. data/lib/aspera/keychain/factory.rb +0 -58
  231. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  232. data/lib/aspera/keychain/macos_security.rb +0 -277
  233. data/lib/aspera/keychain/one_password_api.rb +0 -91
  234. data/lib/aspera/keychain/one_password_base.rb +0 -39
  235. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  236. data/lib/aspera/secret_hider.rb +0 -196
  237. data/lib/aspera/secret_strength.rb +0 -114
@@ -0,0 +1,119 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+
9
+ module Aspera
10
+ module Secret
11
+ # Estimate of the strength of a secret (e.g. a token), in bits: log2 of the number of guesses to find it
12
+ #
13
+ # Uses gem `zxcvbn` if installed (dictionaries, l33t, dates...), else a built-in estimate, the lowest of:
14
+ # - alphabet: length x log2(size of character classes used)
15
+ # - Shannon: empirical entropy of characters (catches repeats)
16
+ # - patterns: repeats, sequences and keyboard rows cost 1 bit per character, words and years 2 bits per character
17
+ # Both under-estimate random secrets: a random secret of 32 bytes (e.g. `openssl rand -hex 32`) is far above any usual minimum.
18
+ module Strength
19
+ # Default minimum strength of tokens, in bits: random tokens of 32 bytes are far above
20
+ DEFAULT_MIN_BITS = 80
21
+ # Classes of characters, with their size
22
+ CHARACTER_CLASSES = [[/[a-z]/, 26], [/[A-Z]/, 26], [/\d/, 10], [/[^a-zA-Z\d]/, 33]].freeze
23
+ # Adjacent characters in those rows form a sequence
24
+ KEYBOARD_ROWS = %w[1234567890 qwertyuiop asdfghjkl zxcvbnm azertyuiop qsdfghjklm wxcvbn].freeze
25
+ # Words (3+ letters, optionally capitalized) and years
26
+ LOW_ENTROPY_REGEXES = [/[A-Z]?[a-z]{3,}/, /(?:19|20)\d\d/].freeze
27
+ # Entropy of English text is 1 to 1.5 bit per letter: keep a margin
28
+ LOW_ENTROPY_BITS = 2.0
29
+ # zxcvbn is slow on long secrets: the beginning is enough to assess strength
30
+ ZXCVBN_MAX_LENGTH = 100
31
+ private_constant :CHARACTER_CLASSES, :KEYBOARD_ROWS, :LOW_ENTROPY_REGEXES, :LOW_ENTROPY_BITS, :ZXCVBN_MAX_LENGTH
32
+
33
+ class << self
34
+ # @param secret [String] Secret to assess
35
+ # @return [Float] Estimated strength in bits
36
+ def bits(secret)
37
+ secret = secret.to_s
38
+ return 0.0 if secret.empty?
39
+ zxcvbn_bits(secret) || estimate_bits(secret)
40
+ end
41
+
42
+ # Refuse an empty or weak secret: an empty one would often accept requests without secret
43
+ # @param secret [String] Secret to check
44
+ # @param name [String] Name of the parameter holding the secret, its minimum is in parameter `<name>_min_bits`
45
+ # @param min_bits [Integer] Minimum estimated strength in bits, `0` checks only that the secret is not empty
46
+ # @param type [Exception] Exception to raise
47
+ # @raise [Exception] if the secret is empty or weaker than `min_bits`
48
+ def assert_strong(secret, name:, min_bits: DEFAULT_MIN_BITS, type: ParameterError)
49
+ Aspera.assert(!secret.to_s.empty?, type: type) { "#{name} must not be empty" }
50
+ strength = bits(secret)
51
+ Aspera.assert(strength >= min_bits, type: type) do
52
+ "#{name} is too weak: about #{strength.round} bits, minimum #{min_bits} (#{name}_min_bits). Generate one with: openssl rand -hex 32"
53
+ end
54
+ end
55
+
56
+ # Built-in estimate, without dictionary
57
+ # @param secret [String] Secret to assess
58
+ # @return [Float] Estimated strength in bits
59
+ def estimate_bits(secret)
60
+ return 0.0 if secret.empty?
61
+ bits_per_char = Math.log2(CHARACTER_CLASSES.sum { |regex, size| secret.match?(regex) ? size : 0 })
62
+ [secret.size * bits_per_char, shannon_bits(secret), pattern_bits(secret, bits_per_char)].min
63
+ end
64
+
65
+ private
66
+
67
+ # @return [Float, nil] Strength estimated by gem `zxcvbn`, or `nil` if not installed
68
+ def zxcvbn_bits(secret)
69
+ return unless zxcvbn?
70
+ Math.log2(Zxcvbn.test(secret[0, ZXCVBN_MAX_LENGTH]).guesses)
71
+ rescue StandardError => e
72
+ Log.log.debug { "zxcvbn failed: #{e.message}" }
73
+ nil
74
+ end
75
+
76
+ # @return [Boolean] `true` if gem `zxcvbn` (or `zxcvbn-ruby`) is installed
77
+ def zxcvbn?
78
+ return @zxcvbn unless @zxcvbn.nil?
79
+ @zxcvbn =
80
+ begin
81
+ require 'zxcvbn'
82
+ true
83
+ rescue LoadError
84
+ false
85
+ end
86
+ end
87
+
88
+ # Empirical Shannon entropy of characters of the secret
89
+ def shannon_bits(secret)
90
+ secret.chars.tally.values.sum do |count|
91
+ probability = count.fdiv(secret.size)
92
+ -probability * Math.log2(probability)
93
+ end * secret.size
94
+ end
95
+
96
+ # Sum of bits of each character, depending on patterns
97
+ def pattern_bits(secret, bits_per_char)
98
+ # Bits of characters in words and years
99
+ low = Array.new(secret.size)
100
+ LOW_ENTROPY_REGEXES.each do |regex|
101
+ secret.scan(regex) { low.fill(LOW_ENTROPY_BITS, Regexp.last_match.begin(0)...Regexp.last_match.end(0)) }
102
+ end
103
+ secret.chars.each_with_index.sum do |char, index|
104
+ next 1.0 if index.positive? && sequence?(secret[index - 1], char)
105
+ low[index] || bits_per_char
106
+ end
107
+ end
108
+
109
+ # @return [Boolean] `true` if `char` repeats `previous`, or follows it in alphabet, digits or keyboard row
110
+ def sequence?(previous, char)
111
+ previous = previous.downcase
112
+ char = char.downcase
113
+ return true if (char.ord - previous.ord).abs <= 1
114
+ KEYBOARD_ROWS.any? { |row| (one = row.index(previous)) && (two = row.index(char)) && (one - two).abs.eql?(1) }
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end
@@ -1,11 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'mime'
7
+ require_relative 'web_server_simple'
3
8
  require 'json'
4
9
  require 'openssl'
5
10
  require 'uri'
6
11
  require 'webrick'
7
- require 'aspera/mime'
8
- require 'aspera/web_server_simple'
9
12
 
10
13
  module Aspera
11
14
  # Mixin for WEBrick servlets of local web servers: JSON responses, and check of requests that a web page could send.
data/lib/aspera/ssh.rb CHANGED
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'log'
3
8
  require 'net/ssh'
4
- require 'aspera/assert'
5
- require 'aspera/log'
6
9
 
7
10
  module Aspera
8
11
  # A simple wrapper around Net::SSH
@@ -16,8 +19,7 @@ module Aspera
16
19
 
17
20
  class << self
18
21
  # Removes ed25519 keys from the default identity list used by Net::SSH.
19
- # Called when the `ed25519` gem is absent or explicitly disabled via
20
- # `ASCLI_ENABLE_ED25519=false`.
22
+ # Called when explicitly disabled via `ASCLI_ENABLE_ED25519=false`.
21
23
  # @return [void]
22
24
  def disable_ed25519_keys
23
25
  Log.log.debug('Disabling SSH ed25519 user keys')
@@ -99,7 +101,7 @@ module Aspera
99
101
  Log.dump(:data, data, level: :trace1)
100
102
  exit_code = data.read_long
101
103
  end
102
- # send command to SSH channel (execute) cspell: disable-next-line
104
+ # send command to SSH channel (execute)
103
105
  channel.send('cexe'.reverse, cmd) do |_ch, success|
104
106
  raise Error, "could not execute command: #{cmd}" unless success
105
107
  channel.send_data(input) unless input.nil?
@@ -118,6 +120,7 @@ module Aspera
118
120
  end
119
121
  end
120
122
 
121
- # Deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
122
- Aspera::Ssh.disable_ed25519_keys if Gem::Specification.find_all_by_name('ed25519').none? || ENV.fetch('ASCLI_ENABLE_ED25519', 'true').eql?('false')
123
+ # Deactivate ed25519 private keys from SSH identities if requested (ed25519 is natively supported by net-ssh 8 using OpenSSL)
124
+ Aspera::Ssh.disable_ed25519_keys if ENV.fetch('ASCLI_ENABLE_ED25519', 'true').eql?('false')
125
+ # Deactivate ecdsa algorithms with JRuby, as it usually causes problems
123
126
  Aspera::Ssh.disable_ecd_sha2_algorithms if defined?(JRUBY_VERSION) && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')
data/lib/aspera/ssl.rb CHANGED
@@ -1,8 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'log'
3
8
  require 'openssl'
4
- require 'aspera/assert'
5
- require 'aspera/log'
6
9
 
7
10
  module Aspera
8
11
  # Give possibility to globally override SSL options
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  class ::String
4
7
  # Transform capitalized to snake case
5
8
  def capital_to_snake
@@ -10,6 +10,7 @@ type: object
10
10
  properties:
11
11
  alt_logdir:
12
12
  type: string
13
+ x-cli-complete: folder
13
14
  x-cli-option: true
14
15
  watchd:
15
16
  type: string
@@ -37,12 +38,14 @@ properties:
37
38
  x-cli-option: true
38
39
  local_dir:
39
40
  type: string
41
+ x-cli-complete: folder
40
42
  x-cli-option: true
41
43
  remote_dir:
42
44
  type: string
43
45
  x-cli-option: true
44
46
  local_db_dir:
45
47
  type: string
48
+ x-cli-complete: folder
46
49
  x-cli-option: true
47
50
  remote_db_dir:
48
51
  type: string
@@ -17,6 +17,7 @@ properties:
17
17
  ascp_dir:
18
18
  description: Directory containing ascp executable to use.
19
19
  type: string
20
+ x-cli-complete: folder
20
21
  assume_no_mods:
21
22
  description: Assume that the directory structure has not been modified.
22
23
  type: boolean
@@ -186,6 +187,7 @@ properties:
186
187
  path:
187
188
  description: The directory to be synchronized on the local host.
188
189
  type: string
190
+ x-cli-complete: folder
189
191
  x-cli-option: --local-dir
190
192
  x-cli-short: -d
191
193
  local_apply_docroot:
@@ -198,6 +200,7 @@ properties:
198
200
  description: Use the specified database directory on the local host.
199
201
  Default is `.private-asp` at the root level of the synchronized directory.
200
202
  type: string
203
+ x-cli-complete: folder
201
204
  x-cli-option: true
202
205
  x-cli-short: -b
203
206
  local_checksum_threads:
@@ -228,6 +231,7 @@ properties:
228
231
  local_keep_dir:
229
232
  description: Move deleted files into the specified directory on the local host.
230
233
  type: string
234
+ x-cli-complete: folder
231
235
  x-cli-option: --keep-dir-local
232
236
  local_mount_signature:
233
237
  description: Verify that the file system is mounted by the existence of this file on the local host.
@@ -296,6 +300,7 @@ properties:
296
300
  local_dir:
297
301
  description: Use the specified logging directory on the local host.
298
302
  type: string
303
+ x-cli-complete: folder
299
304
  x-cli-option: --alt-logdir
300
305
  x-cli-short: -L
301
306
  remote_dir:
@@ -692,11 +697,11 @@ properties:
692
697
  `vhigh` maximizes system activity by continuously scanning files without rest.
693
698
  type: string
694
699
  enum:
695
- - vlow
700
+ - vlow # cspell:disable-line
696
701
  - low
697
702
  - medium
698
703
  - high
699
- - vhigh
704
+ - vhigh # cspell:disable-line
700
705
  default: medium
701
706
  x-cli-option: true
702
707
  x-cli-short: -H
@@ -806,7 +811,7 @@ properties:
806
811
  type: integer
807
812
  x-cli-option: true
808
813
  x-cli-short: -g
809
- rexmsg_size:
814
+ rexmsg_size: # cspell:disable-line
810
815
  description:
811
816
  Use the specified size (in bytes) for a retransmission request.
812
817
  Default is determined by `aspera.conf`.
@@ -846,7 +851,7 @@ properties:
846
851
  enum:
847
852
  - none
848
853
  - redis
849
- - scalekv
854
+ - scalekv # cspell:disable-line
850
855
  default: none
851
856
  domain:
852
857
  description: Specify the domain.
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  if defined?(JRUBY_VERSION)
4
7
  require 'jdbc/sqlite3'
5
8
  Jdbc::SQLite3.load_driver
@@ -1,18 +1,20 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:words logdir bidi watchd cooloff asyncadmin
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
4
5
 
5
- require 'aspera/ascp/installation'
6
- require 'aspera/agent/direct'
7
- require 'aspera/transfer/spec'
8
- require 'aspera/exec_spec'
9
- require 'aspera/command_line_converter'
10
- require 'aspera/command_line_builder'
11
- require 'aspera/log'
12
- require 'aspera/assert'
13
- require 'aspera/duration'
14
- require 'aspera/environment'
15
- require 'aspera/schema/registry'
6
+ require_relative '../ascp/installation'
7
+ require_relative '../agent/direct'
8
+ require_relative '../transfer/spec'
9
+ require_relative '../transfer/units'
10
+ require_relative '../exec_spec'
11
+ require_relative '../command_line_converter'
12
+ require_relative '../command_line_builder'
13
+ require_relative '../log'
14
+ require_relative '../assert'
15
+ require_relative '../duration'
16
+ require_relative '../environment'
17
+ require_relative '../schema/registry'
16
18
  require 'json'
17
19
  require 'base64'
18
20
  require 'open3'
@@ -266,7 +268,7 @@ module Aspera
266
268
  # Convert `target_rate` given as a string with optional suffix (e.g. `100m`) into integer bps
267
269
  # @param params [Hash, nil] `transport` (`conf` format) or session (`args` format), modified in place
268
270
  def normalize_target_rate(params)
269
- params['target_rate'] = Transfer::Spec.rate_string_to_bps(params['target_rate']) if params.is_a?(Hash) && params['target_rate'].is_a?(String)
271
+ params['target_rate'] = Transfer::Units.rate_string_to_bps(params['target_rate']) if params.is_a?(Hash) && params['target_rate'].is_a?(String)
270
272
  end
271
273
 
272
274
  # Transfer specification to synchronization information
@@ -363,7 +365,7 @@ module Aspera
363
365
  CONF_SCHEMA = CommandLineBuilder.read_schema(Schema::Registry::SYNC_CONF)
364
366
  CMDLINE_PARAMS_KEYS = %w[instance sessions].freeze
365
367
  ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
366
- PRIVATE_FOLDER = "#{Environment.instance.os.eql?(Environment::OS_WINDOWS) ? '~' : '.'}private-asp"
368
+ PRIVATE_FOLDER = "#{Environment.instance.os.eql?(Environment::OS_WINDOWS) ? '~' : '.'}private-asp".freeze
367
369
  ASYNC_DB = 'snap.db'
368
370
  PARAM_KEYS = %w[local sessions].freeze
369
371
 
@@ -1,9 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
4
- require 'aspera/log'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative 'assert'
7
+ require_relative 'log'
5
8
  require 'singleton'
6
9
  require 'fileutils'
10
+ require 'securerandom'
7
11
  require 'etc'
8
12
  require 'tmpdir'
9
13
 
@@ -114,7 +118,7 @@ module Aspera
114
118
  def new_file_path_global(prefix = nil, suffix: nil)
115
119
  username =
116
120
  begin
117
- Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
121
+ Etc.getlogin || Etc.getpwuid(::Process.uid).name || 'unknown_user'
118
122
  rescue StandardError
119
123
  'unknown_user'
120
124
  end
@@ -128,7 +132,7 @@ module Aspera
128
132
  def new_dir_path_global(prefix = nil)
129
133
  username =
130
134
  begin
131
- Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'
135
+ Etc.getlogin || Etc.getpwuid(::Process.uid).name || 'unknown_user'
132
136
  rescue StandardError
133
137
  'unknown_user'
134
138
  end
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  # trigger returns true only if the delay has passed since the last trigger
5
8
  class TimerLimiter
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/ascp/management'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../ascp/management'
4
7
 
5
8
  module Aspera
6
9
  module Transfer
@@ -1,12 +1,15 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  module Transfer
5
8
  # Generates a pseudo file stream.
6
9
  class FauxFile
7
10
  SCHEME = 'faux'
8
11
  # marker for faux file
9
- PREFIX = "#{SCHEME}:///"
12
+ PREFIX = "#{SCHEME}:///".freeze
10
13
  # size units, kilo, mega ...
11
14
  SIZE_UNITS = %w[k m g t p e].freeze
12
15
  private_constant :SCHEME, :PREFIX, :SIZE_UNITS
@@ -1,22 +1,25 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
- require 'aspera/assert'
5
- require 'aspera/exec_spec'
6
- require 'aspera/command_line_builder'
7
- require 'aspera/temp_file_manager'
8
- require 'aspera/transfer/error'
9
- require 'aspera/transfer/spec'
10
- require 'aspera/command_line_converter'
11
- require 'aspera/ascp/installation'
12
- require 'aspera/cli/formatter'
13
- require 'aspera/rest'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+ require_relative '../exec_spec'
9
+ require_relative '../command_line_builder'
10
+ require_relative '../temp_file_manager'
11
+ require_relative 'error'
12
+ require_relative 'spec'
13
+ require_relative '../command_line_converter'
14
+ require_relative '../ascp/installation'
15
+ require_relative '../cli/formatter'
16
+ require_relative '../rest'
17
+ require_relative '../rainbow'
14
18
  require 'securerandom'
15
19
  require 'base64'
16
20
  require 'json'
17
21
  require 'fileutils'
18
22
  require 'openssl'
19
- require 'aspera/rainbow'
20
23
  using Rainbow
21
24
 
22
25
  module Aspera
@@ -89,7 +92,7 @@ module Aspera
89
92
  # is the file list provided through ascp parameters?
90
93
  ascp_file_list_provided = self.class.ascp_args_file_list?(@ascp_args)
91
94
  # set if paths is mandatory in ts
92
- @builder.required('paths', !@job_spec.key?('keepalive') && !ascp_file_list_provided) # cspell:words keepalive
95
+ @builder.required('paths', !@job_spec.key?('keepalive') && !ascp_file_list_provided)
93
96
  # get paths in transfer spec (after setting if it is mandatory)
94
97
  ts_paths_array = @builder.read_param('paths')
95
98
  file_list_option = nil
@@ -221,7 +224,7 @@ module Aspera
221
224
  Log.log.debug { "ascp args: #{exec_spec}" }
222
225
  return exec_spec
223
226
  end
224
- DELETE_EQUIV = %w[remove_after_transfer remove_empty_directories remove_empty_source_directory]
227
+ DELETE_EQUIV = %w[remove_after_transfer remove_empty_directories remove_empty_source_directory].freeze
225
228
  # `ascp` options to provide a file list
226
229
  FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
227
230
  private_constant :DELETE_EQUIV, :FILE_LIST_OPTIONS
@@ -1,5 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
3
6
  module Aspera
4
7
  module Transfer
5
8
  # Typed result of a transfer operation.
@@ -1,9 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'singleton'
4
- require 'aspera/log'
5
- require 'aspera/assert'
6
- require 'aspera/transfer/error'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../log'
7
+ require_relative '../assert'
8
+ require_relative 'error'
7
9
 
8
10
  module Aspera
9
11
  module Transfer
@@ -1,8 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/agent/factory'
4
- require 'aspera/command_line_builder'
5
- require 'aspera/assert'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../agent/factory'
7
+ require_relative '../command_line_builder'
8
+ require_relative '../assert'
9
+ require_relative 'units'
6
10
 
7
11
  module Aspera
8
12
  module Transfer
@@ -37,14 +41,8 @@ module Aspera
37
41
  'attrs' => 'attributes',
38
42
  'sparse_csum' => 'sparse_checksum',
39
43
  'full_csum' => 'full_checksum'
40
- }
41
- # Multipliers for rate suffix (result is in bps)
42
- RATE_SUFFIX_BPS = {
43
- 'k' => 1_000,
44
- 'm' => 1_000_000,
45
- 'g' => 1_000_000_000
46
44
  }.freeze
47
- private_constant :POLICY_FIX, :RATE_SUFFIX_BPS
45
+ private_constant :POLICY_FIX
48
46
  # translate upload/download to send/receive
49
47
  def transfer_type_to_direction(transfer_type)
50
48
  XFER_TYPE_TO_DIR.fetch(transfer_type)
@@ -55,23 +53,12 @@ module Aspera
55
53
  XFER_DIR_TO_TYPE.fetch(direction)
56
54
  end
57
55
 
58
- # Parse a human-readable rate (as accepted by async -l) into an integer bps value.
59
- # Accepted: plain integer (bps), or integer + suffix k/K (x1000), m/M (x1000000), g/G (x1000000000).
60
- # @param value [String, Integer] e.g. "100m", "500000", "1g", 500000
61
- # @return [Integer] value in bps
62
- def rate_string_to_bps(value)
63
- m = value.to_s.strip.match(/\A(\d+)([kKmMgG])?\z/)
64
- Aspera.assert(m) { "Invalid rate value: #{value.inspect}. Expected integer with optional suffix k/K, m/M or g/G." }
65
- multiplier = m[2] ? RATE_SUFFIX_BPS.fetch(m[2].downcase) : 1
66
- return m[1].to_i * multiplier
67
- end
68
-
69
56
  # Resolve pseudo-parameter `target_rate` (bps) into `target_rate_kbps` (rounded down), overriding it if both are present.
70
57
  # @param transfer_spec [Hash] Transfer spec, modified in place
71
58
  def resolve_target_rate(transfer_spec)
72
59
  return unless transfer_spec.key?('target_rate')
73
60
  rate = transfer_spec.delete('target_rate')
74
- transfer_spec['target_rate_kbps'] = rate_string_to_bps(rate) / 1000 unless rate.nil?
61
+ transfer_spec['target_rate_kbps'] = Units.rate_string_to_bps(rate) / 1000 unless rate.nil?
75
62
  end
76
63
 
77
64
  def fix_transferd_resume_policy(transfer_spec)
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../assert'
7
+
8
+ module Aspera
9
+ module Transfer
10
+ # Rates, sizes and durations of transfers: parsed from user input and formatted for display.
11
+ # Decimal units, like `ascp`. Depends only on `aspera/assert`: usable without the CLI.
12
+ module Units
13
+ # Decimal prefixes: each one is `KILO` times the previous one
14
+ PREFIX = ['', 'K', 'M', 'G', 'T', 'P'].freeze
15
+ KILO = 1000
16
+ # Multiplier of each prefix, e.g. `'M' => 1_000_000`
17
+ MULTIPLIER = PREFIX.each_with_index.to_h { |prefix, power| [prefix, KILO**power] }.freeze
18
+ # Units of sizes: `B`, `KB`, `MB`...
19
+ BYTE_UNITS = PREFIX.map { |prefix| "#{prefix}B" }.freeze
20
+ # Rate typed by a user: integer with an optional prefix, case ignored
21
+ RATE_INPUT = /\A(\d+)([#{PREFIX.join}])?\z/i
22
+ BITS_PER_BYTE = 8
23
+ private_constant :MULTIPLIER, :BYTE_UNITS, :RATE_INPUT, :BITS_PER_BYTE
24
+
25
+ class << self
26
+ # Parse a human-readable rate (as accepted by async -l) into an integer bps value.
27
+ # Accepted: plain integer (bps), or integer + prefix of `PREFIX`, case ignored, e.g. `k` (x1000), `M` (x1000000), `g` (x1000000000).
28
+ # @param value [String, Integer] e.g. "100m", "500000", "1g", 500000
29
+ # @return [Integer] value in bps
30
+ def rate_string_to_bps(value)
31
+ m = RATE_INPUT.match(value.to_s.strip)
32
+ Aspera.assert(m) { "Invalid rate value: #{value.inspect}. Expected integer with optional suffix #{PREFIX.drop(1).join(', ')} (case ignored)." }
33
+ return m[1].to_i * MULTIPLIER.fetch(m[2].to_s.upcase)
34
+ end
35
+
36
+ # Inverse of `rate_string_to_bps`: the shortest exact text, e.g. `100M`, `1G`, `1500K`, `999`
37
+ # @param bps [Integer] Rate in bits per second
38
+ # @return [String] Rate accepted by `rate_string_to_bps`
39
+ def rate_bps_to_string(bps)
40
+ prefix = PREFIX.reverse.find { |candidate| bps.positive? && (bps % MULTIPLIER[candidate]).zero? } || ''
41
+ "#{bps / MULTIPLIER[prefix]}#{prefix}"
42
+ end
43
+
44
+ # @param bytes [Integer] Size in bytes
45
+ # @return [String] Size with decimal unit, e.g. `12.3 MB`
46
+ def format_bytes(bytes)
47
+ value = bytes.to_f
48
+ index = 0
49
+ while value >= KILO && index < BYTE_UNITS.length - 1
50
+ value /= KILO
51
+ index += 1
52
+ end
53
+ return "#{bytes} B" if index.zero?
54
+ format('%.1f %s', value, BYTE_UNITS[index])
55
+ end
56
+
57
+ # @param bytes_per_second [Float, nil] Rate, `nil` if unknown
58
+ # @return [String] Rate in megabits per second (decimal, like `ascp`)
59
+ def format_rate(bytes_per_second)
60
+ return '-- Mbps' if bytes_per_second.nil?
61
+ mbps = bytes_per_second * BITS_PER_BYTE / MULTIPLIER.fetch('M')
62
+ mbps < 10 ? format('%.2f Mbps', mbps) : format('%.0f Mbps', mbps)
63
+ end
64
+
65
+ # @param seconds [Numeric, nil] Duration, `nil` if unknown
66
+ # @return [String] Duration as `hh:mm:ss`
67
+ def format_duration(seconds)
68
+ return '--:--:--' if seconds.nil?
69
+ seconds = seconds.round
70
+ format('%02d:%02d:%02d', seconds / 3600, (seconds / 60) % 60, seconds % 60)
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end