aspera-cli 4.26.2 → 4.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +310 -21
  4. data/CONTRIBUTING.md +99 -14
  5. data/TODO.md +50 -0
  6. data/bin/ascli +3 -3
  7. data/docs/README.md +10977 -0
  8. data/docs/test-mcp-with-ai.md +202 -0
  9. data/lib/aspera/agent/base.rb +29 -7
  10. data/lib/aspera/agent/connect.rb +58 -21
  11. data/lib/aspera/agent/desktop.rb +53 -18
  12. data/lib/aspera/agent/direct.rb +71 -40
  13. data/lib/aspera/agent/factory.rb +3 -3
  14. data/lib/aspera/agent/httpgw.rb +55 -14
  15. data/lib/aspera/agent/node.rb +34 -2
  16. data/lib/aspera/agent/transferd.rb +48 -12
  17. data/lib/aspera/api/alee.rb +1 -1
  18. data/lib/aspera/api/aoc.rb +42 -44
  19. data/lib/aspera/api/cos_node.rb +10 -9
  20. data/lib/aspera/api/faspex.rb +16 -15
  21. data/lib/aspera/api/httpgw.rb +23 -23
  22. data/lib/aspera/api/node.rb +51 -70
  23. data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
  24. data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
  25. data/lib/aspera/ascmd.rb +32 -30
  26. data/lib/aspera/ascp/installation.rb +75 -69
  27. data/lib/aspera/ascp/management.rb +7 -8
  28. data/lib/aspera/assert.rb +12 -13
  29. data/lib/aspera/cli/ascp_actions.rb +155 -0
  30. data/lib/aspera/cli/async_transfer_store.rb +125 -0
  31. data/lib/aspera/cli/bootstrapper.rb +197 -0
  32. data/lib/aspera/cli/command_registry.rb +145 -0
  33. data/lib/aspera/cli/command_spec.rb +141 -0
  34. data/lib/aspera/cli/context.rb +19 -6
  35. data/lib/aspera/cli/error.rb +16 -0
  36. data/lib/aspera/cli/extended_value.rb +27 -26
  37. data/lib/aspera/cli/formatter.rb +95 -59
  38. data/lib/aspera/cli/gem_checker.rb +65 -0
  39. data/lib/aspera/cli/hints.rb +3 -3
  40. data/lib/aspera/cli/http.rb +74 -30
  41. data/lib/aspera/cli/info.rb +2 -0
  42. data/lib/aspera/cli/mailer.rb +97 -0
  43. data/lib/aspera/cli/mcp_tool.rb +225 -0
  44. data/lib/aspera/cli/option_declarator.rb +77 -0
  45. data/lib/aspera/cli/options.schema.yaml +634 -7
  46. data/lib/aspera/cli/parser.rb +1217 -0
  47. data/lib/aspera/cli/plugins/alee.rb +20 -22
  48. data/lib/aspera/cli/plugins/aoc.rb +1178 -881
  49. data/lib/aspera/cli/plugins/ats.rb +205 -162
  50. data/lib/aspera/cli/plugins/base.rb +703 -175
  51. data/lib/aspera/cli/plugins/basic_auth.rb +7 -9
  52. data/lib/aspera/cli/plugins/config.rb +434 -757
  53. data/lib/aspera/cli/plugins/console.rb +107 -65
  54. data/lib/aspera/cli/plugins/cos.rb +46 -33
  55. data/lib/aspera/cli/plugins/factory.rb +6 -6
  56. data/lib/aspera/cli/plugins/faspex5.rb +627 -388
  57. data/lib/aspera/cli/plugins/faspio.rb +54 -51
  58. data/lib/aspera/cli/plugins/httpgw.rb +18 -25
  59. data/lib/aspera/cli/plugins/mcp.rb +279 -0
  60. data/lib/aspera/cli/plugins/node.rb +988 -840
  61. data/lib/aspera/cli/plugins/oauth.rb +7 -10
  62. data/lib/aspera/cli/plugins/orchestrator.rb +112 -135
  63. data/lib/aspera/cli/plugins/preview.rb +195 -157
  64. data/lib/aspera/cli/plugins/server.rb +141 -91
  65. data/lib/aspera/cli/plugins/shares.rb +343 -110
  66. data/lib/aspera/cli/preset_actions.rb +159 -0
  67. data/lib/aspera/cli/preset_manager.rb +82 -39
  68. data/lib/aspera/cli/result.rb +78 -26
  69. data/lib/aspera/cli/runner.rb +323 -153
  70. data/lib/aspera/cli/secret_finder.rb +40 -0
  71. data/lib/aspera/cli/special_values.rb +1 -0
  72. data/lib/aspera/cli/sync_actions.rb +89 -64
  73. data/lib/aspera/cli/terminal_formatter.rb +3 -3
  74. data/lib/aspera/cli/transfer_actions.rb +90 -0
  75. data/lib/aspera/cli/transfer_agent.rb +129 -64
  76. data/lib/aspera/cli/transfer_progress.rb +9 -9
  77. data/lib/aspera/cli/vault_manager.rb +79 -0
  78. data/lib/aspera/cli/version.rb +1 -1
  79. data/lib/aspera/cli/wizard.rb +28 -30
  80. data/lib/aspera/colors.rb +3 -3
  81. data/lib/aspera/command_line_builder.rb +34 -34
  82. data/lib/aspera/command_line_converter.rb +1 -1
  83. data/lib/aspera/coverage.rb +1 -2
  84. data/lib/aspera/data_repository.rb +2 -1
  85. data/lib/aspera/dot_container.rb +12 -12
  86. data/lib/aspera/environment.rb +30 -23
  87. data/lib/aspera/exec_spec.rb +13 -0
  88. data/lib/aspera/faspex_gw.rb +5 -5
  89. data/lib/aspera/faspex_postproc.rb +16 -10
  90. data/lib/aspera/graphql.rb +37 -0
  91. data/lib/aspera/hash_ext.rb +8 -2
  92. data/lib/aspera/json_rpc/client.rb +62 -0
  93. data/lib/aspera/json_rpc/version.rb +7 -0
  94. data/lib/aspera/keychain/base.rb +10 -3
  95. data/lib/aspera/keychain/encrypted_hash.rb +92 -17
  96. data/lib/aspera/keychain/factory.rb +20 -8
  97. data/lib/aspera/keychain/hashicorp_vault.rb +1 -1
  98. data/lib/aspera/keychain/macos_security.rb +23 -25
  99. data/lib/aspera/keychain/one_password_api.rb +86 -0
  100. data/lib/aspera/keychain/one_password_base.rb +34 -0
  101. data/lib/aspera/keychain/one_password_cli.rb +98 -0
  102. data/lib/aspera/link_header.rb +82 -0
  103. data/lib/aspera/log.rb +29 -15
  104. data/lib/aspera/markdown.rb +90 -7
  105. data/lib/aspera/nagios.rb +8 -8
  106. data/lib/aspera/node_simulator.rb +35 -27
  107. data/lib/aspera/oauth/base.rb +9 -10
  108. data/lib/aspera/oauth/boot.rb +5 -5
  109. data/lib/aspera/oauth/factory.rb +12 -10
  110. data/lib/aspera/oauth/jwt.rb +9 -11
  111. data/lib/aspera/oauth/web.rb +6 -6
  112. data/lib/aspera/persistency_action_once.rb +12 -12
  113. data/lib/aspera/persistency_folder.rb +18 -11
  114. data/lib/aspera/preview/file_types.rb +5 -5
  115. data/lib/aspera/preview/generator.rb +52 -49
  116. data/lib/aspera/preview/options.rb +3 -2
  117. data/lib/aspera/preview/terminal.rb +5 -5
  118. data/lib/aspera/preview/utils.rb +66 -19
  119. data/lib/aspera/products/connect.rb +2 -2
  120. data/lib/aspera/products/desktop.rb +1 -1
  121. data/lib/aspera/products/other.rb +3 -3
  122. data/lib/aspera/products/transferd.rb +4 -4
  123. data/lib/aspera/proxy_auto_config.rb +15 -14
  124. data/lib/aspera/rest.rb +159 -151
  125. data/lib/aspera/rest_error_analyzer.rb +6 -6
  126. data/lib/aspera/rest_errors_aspera.rb +0 -10
  127. data/lib/aspera/rest_list.rb +11 -10
  128. data/lib/aspera/schema/IBM Aspera Node API-4.4.6.yaml +6232 -0
  129. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +249 -0
  130. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
  131. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +4653 -0
  132. data/lib/aspera/schema/documentation.rb +65 -28
  133. data/lib/aspera/schema/reader.rb +136 -13
  134. data/lib/aspera/schema/registry.rb +60 -8
  135. data/lib/aspera/secret_hider.rb +12 -7
  136. data/lib/aspera/ssh.rb +64 -31
  137. data/lib/aspera/ssl.rb +5 -5
  138. data/lib/aspera/sync/conf.schema.yaml +2 -2
  139. data/lib/aspera/sync/database.rb +2 -2
  140. data/lib/aspera/sync/operations.rb +22 -25
  141. data/lib/aspera/temp_file_manager.rb +25 -5
  142. data/lib/aspera/timer_limiter.rb +1 -1
  143. data/lib/aspera/transfer/faux_file.rb +24 -11
  144. data/lib/aspera/transfer/parameters.rb +36 -34
  145. data/lib/aspera/transfer/result.rb +74 -0
  146. data/lib/aspera/transfer/resumer.rb +10 -10
  147. data/lib/aspera/transfer/spec.rb +18 -1
  148. data/lib/aspera/transfer/spec.schema.yaml +12 -3
  149. data/lib/aspera/uri_reader.rb +52 -15
  150. data/lib/aspera/web_auth.rb +8 -7
  151. data/lib/aspera/web_server_simple.rb +15 -12
  152. data/lib/aspera/yaml.rb +5 -4
  153. data.tar.gz.sig +0 -0
  154. metadata +34 -6
  155. metadata.gz.sig +0 -0
  156. data/lib/aspera/cli/manager.rb +0 -766
  157. data/lib/aspera/cli/plugins/faspex.rb +0 -539
  158. data/lib/aspera/json_rpc.rb +0 -52
  159. data/lib/aspera/transfer/uri.rb +0 -56
@@ -48,10 +48,10 @@ module Aspera
48
48
  attr_reader :path_token
49
49
 
50
50
  # Helper method to create token as per RFC
51
- # @return [HTTPResponse]
52
- # @raise RestError if not 2XX code
51
+ # @return [Net::HTTPResponse] raw HTTP response with token
52
+ # @raise [RestCallError] if not 2XX code
53
53
  def create_token_call(creation_params)
54
- Log.log.debug{'Generating a new token'.bg_green}
54
+ Log.log.debug { 'Generating a new token'.bg_green }
55
55
  return @api.create(@path_token, nil, query: creation_params, ret: :resp) if @use_query
56
56
  return @api.create(@path_token, creation_params, content_type: Mime::WWW, ret: :resp)
57
57
  end
@@ -73,8 +73,8 @@ module Aspera
73
73
  # get an OAuth v2 token (generated, cached, refreshed)
74
74
  # call token() to get a token.
75
75
  # if a token is expired (api returns 4xx), call again token(refresh: true)
76
- # @param cache set to false to disable cache
77
- # @param refresh set to true to force refresh or re-generation (if previous failed)
76
+ # @param cache [Boolean] set to false to disable cache
77
+ # @param refresh [Boolean] set to true to force refresh or re-generation (if previous failed)
78
78
  def token(cache: true, refresh: false)
79
79
  # get token info from cache (or nil), decoded with date and expiration status
80
80
  token_info = Factory.instance.get_token_info(@token_cache_id) if cache
@@ -87,7 +87,7 @@ module Aspera
87
87
  # `direct` agent is equipped with refresh code
88
88
  # an API was already called, but failed, we need to regenerate or refresh
89
89
  if refresh || token_info[:expired]
90
- Log.log.trace1{"refresh: #{refresh} expired: #{token_info[:expired]}"}
90
+ Log.log.trace1 { "refresh: #{refresh} expired: #{token_info[:expired]}" }
91
91
  refresh_token = nil
92
92
  if token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
93
93
  # save possible refresh token, before deleting the cache
@@ -99,16 +99,15 @@ module Aspera
99
99
  # lets try the existing refresh token
100
100
  # NOTE: AoC admin token has no refresh, and lives by default 1800secs
101
101
  if !refresh_token.nil?
102
- Log.log.debug{"refresh token=[#{refresh_token}]"}
103
102
  begin
104
103
  http = create_token_call(base_params(add_secret: true).merge(grant_type: 'refresh_token', refresh_token: refresh_token))
105
104
  # Save only if success
106
105
  json_data = http.body
107
106
  token_data = JSON.parse(json_data)
108
107
  Factory.instance.persist_mgr.put(@token_cache_id, json_data)
109
- rescue => e
108
+ rescue StandardError => e
110
109
  # Refresh token can fail.
111
- Log.log.warn{"Refresh failed: #{e}"}
110
+ Log.log.warn { "Refresh failed: #{e}" }
112
111
  end
113
112
  end
114
113
  end
@@ -122,7 +121,7 @@ module Aspera
122
121
  token_data = JSON.parse(json_data)
123
122
  Factory.instance.persist_mgr.put(@token_cache_id, json_data)
124
123
  end
125
- Aspera.assert(token_data.key?(@token_field)){"API error: No such field in answer: #{@token_field}"} unless token_data.nil?
124
+ Aspera.assert(token_data.key?(@token_field)) { "API error: No such field in answer: #{@token_field}" } unless token_data.nil?
126
125
  # ok we shall have a token here
127
126
  return token_data[@token_field]
128
127
  end
@@ -8,24 +8,24 @@ module Aspera
8
8
  module OAuth
9
9
  # Token provider bootstrapped from an existing cookie (e.g. AoC browser cookie).
10
10
  # Injects the bearer token and optional refresh token directly into the cache.
11
- # Never generates a new token from scratch raises if cache+refresh are both exhausted.
11
+ # Never generates a new token from scratch - raises if cache+refresh are both exhausted.
12
12
  class Boot < Base
13
13
  # @param cookie [String, nil] Raw cookie string (--password), nil to rely on existing cache
14
14
  # @param username [String, nil] Expected subject; if provided, must match token's `sub` claim
15
- # @param **base_params Forwarded to Base (base_url:, params: {client_id:, scope:}, etc.)
15
+ # @param **base_params [Hash] Forwarded to Base (base_url:, params: {client_id:, scope:}, etc.)
16
16
  def initialize(cookie: nil, username: nil, **base_params)
17
17
  if cookie.nil?
18
18
  # No cookie: rely on existing cache, identified by username if provided
19
19
  Aspera.assert(username, 'Provide --password (cookie) on first use, or --username for cache lookup', type: ParameterError)
20
20
  super(**base_params, cache_ids: [username])
21
21
  else
22
- cookies = cookie.split('; ').map{ |p| p.split('=', 2)}.to_h
22
+ cookies = cookie.split('; ').map { |p| p.split('=', 2) }.to_h
23
23
  Aspera.assert(cookies.key?('aoc.token'), '--password cookie does not contain aoc.token', type: ParameterError)
24
24
  token = cookies['aoc.token']
25
25
  decoded = Factory.instance.decode_token(token)
26
- Aspera.assert_type(decoded, Hash){'Boot: token is not a decodable JWT'}
26
+ Aspera.assert_type(decoded, Hash) { 'Boot: token is not a decodable JWT' }
27
27
  sub = decoded['sub']
28
- Aspera.assert(username.nil? || username.eql?(sub)){"Boot: --username #{username} does not match token subject #{sub}"}
28
+ Aspera.assert(username.nil? || username.eql?(sub)) { "Boot: --username #{username} does not match token subject #{sub}" }
29
29
  super(**base_params, cache_ids: [sub])
30
30
  token_data = {Factory::TOKEN_FIELD => token}
31
31
  token_data['refresh_token'] = cookies['aoc.refresh'] if cookies.key?('aoc.refresh')
@@ -41,7 +41,7 @@ module Aspera
41
41
  # @param authorization [String] The authorization header value
42
42
  # @return [String] The bearer token without the scheme prefix
43
43
  def bearer_token(authorization)
44
- Aspera.assert(bearer_auth?(authorization)){'not a bearer token, wrong prefix scheme'}
44
+ Aspera.assert(bearer_auth?(authorization)) { 'not a bearer token, wrong prefix scheme' }
45
45
  return authorization.delete_prefix(SPACE_BEARER_AUTH_SCHEME)
46
46
  end
47
47
 
@@ -108,7 +108,7 @@ module Aspera
108
108
  end
109
109
 
110
110
  # Delete all existing tokens in cache
111
- # @return [void]
111
+ # @return [Array<String>] List of deleted file paths
112
112
  def flush_tokens
113
113
  persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN)
114
114
  end
@@ -119,7 +119,7 @@ module Aspera
119
119
  data = persist_mgr.current_items(PERSIST_CATEGORY_TOKEN)
120
120
  data.each.map do |k, v|
121
121
  info = {id: k}
122
- info.merge!(JSON.parse(v)) rescue nil
122
+ begin; info.merge!(JSON.parse(v)); rescue StandardError; nil; end
123
123
  d = decode_token(info.delete(TOKEN_FIELD))
124
124
  info.merge(d) if d
125
125
  info
@@ -155,9 +155,10 @@ module Aspera
155
155
 
156
156
  # Register a bearer token decoder for inspecting token properties
157
157
  # @param method [Proc] The decoder lambda/proc to register
158
- # @return [void]
158
+ # @return [nil]
159
159
  def register_decoder(method)
160
160
  @decoders.push(method)
161
+ nil
161
162
  end
162
163
 
163
164
  # Decode a token using all registered decoders
@@ -165,7 +166,7 @@ module Aspera
165
166
  # @return [Hash, nil] Decoded token data or nil if no decoder succeeded
166
167
  def decode_token(token)
167
168
  @decoders.each do |decoder|
168
- result = decoder.call(token) rescue nil
169
+ result = begin; decoder.call(token); rescue StandardError; nil; end
169
170
  return result unless result.nil?
170
171
  end
171
172
  return
@@ -173,12 +174,13 @@ module Aspera
173
174
 
174
175
  # Register a token creation method
175
176
  # @param creator_class [Class] The token creator class to register
176
- # @return [void]
177
+ # @return [nil]
177
178
  def register_token_creator(creator_class)
178
179
  Aspera.assert_type(creator_class, Class)
179
180
  id = Factory.class_to_id(creator_class)
180
- Log.log.debug{"registering creator for #{id}"}
181
+ Log.log.debug { "registering creator for #{id}" }
181
182
  @token_type_classes[id] = creator_class
183
+ nil
182
184
  end
183
185
 
184
186
  # Create a token creator instance for the specified grant method
@@ -187,12 +189,12 @@ module Aspera
187
189
  def create(**parameters)
188
190
  Aspera.assert_type(parameters, Hash)
189
191
  id = parameters[:grant_method]
190
- Aspera.assert(@token_type_classes.key?(id)){"token grant method unknown: '#{id}'"}
191
- create_parameters = parameters.reject{ |k, _v| k.eql?(:grant_method)}
192
+ Aspera.assert(@token_type_classes.key?(id)) { "token grant method unknown: '#{id}'" }
193
+ create_parameters = parameters.reject { |k, _v| k.eql?(:grant_method) }
192
194
  @token_type_classes[id].new(**create_parameters)
193
195
  end
194
196
  end
195
197
  # JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
196
- Factory.instance.register_decoder(lambda{ |token| parts = token.split('.'); Aspera.assert_values(parts.length, [3]){'JWS token parts'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon
198
+ Factory.instance.register_decoder(lambda { |token| parts = token.split('.'); Aspera.assert_values(parts.length, [3]) { 'JWS token parts' }; JSON.parse(Base64.decode64(parts[1])) }) # rubocop:disable Style/Semicolon
197
199
  end
198
200
  end
@@ -26,9 +26,9 @@ module Aspera
26
26
  end
27
27
  DEFAULT_PRIV_KEY_LENGTH = 4096
28
28
  GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
29
- # @param private_key_obj private key object
30
- # @param payload payload to be included in the JWT
31
- # @param headers headers to be included in the JWT
29
+ # @param private_key_obj [OpenSSL::PKey::RSA] private key object
30
+ # @param payload [Hash] payload to be included in the JWT
31
+ # @param headers [Hash] headers to be included in the JWT
32
32
  def initialize(
33
33
  private_key_obj:,
34
34
  payload:,
@@ -36,10 +36,10 @@ module Aspera
36
36
  cache_ids: [],
37
37
  **base_params
38
38
  )
39
- Aspera.assert_type(private_key_obj, OpenSSL::PKey::RSA){'private_key_obj'}
40
- Aspera.assert_type(payload, Hash){'payload'}
41
- Aspera.assert_type(headers, Hash){'headers'}
42
- Aspera.assert_type(cache_ids, Array){'cache ids'}
39
+ Aspera.assert_type(private_key_obj, OpenSSL::PKey::RSA) { 'private_key_obj' }
40
+ Aspera.assert_type(payload, Hash) { 'payload' }
41
+ Aspera.assert_type(headers, Hash) { 'headers' }
42
+ Aspera.assert_type(cache_ids, Array) { 'cache ids' }
43
43
  new_cache_ids = cache_ids.clone.push(payload[:sub])
44
44
  super(**base_params, cache_ids: new_cache_ids)
45
45
  @private_key_obj = private_key_obj
@@ -49,8 +49,8 @@ module Aspera
49
49
 
50
50
  def create_token
51
51
  require 'jwt'
52
- seconds_since_epoch = Time.new.to_i
53
- Log.log.debug{"seconds_since_epoch=#{seconds_since_epoch}"}
52
+ seconds_since_epoch = Time.now.to_i
53
+ Log.log.debug { "seconds_since_epoch=#{seconds_since_epoch}" }
54
54
  jwt_payload = {
55
55
  exp: seconds_since_epoch + OAuth::Factory.instance.parameters[:jwt_expiry_offset_sec], # expiration time
56
56
  nbf: seconds_since_epoch - OAuth::Factory.instance.parameters[:jwt_accepted_offset_sec], # not before
@@ -58,9 +58,7 @@ module Aspera
58
58
  jti: SecureRandom.uuid # JWT id
59
59
  }.merge(@additional_payload)
60
60
  Log.dump(:jwt_payload, jwt_payload)
61
- Log.log.debug{"private=[#{@private_key_obj}]"}
62
61
  assertion = JWT.encode(jwt_payload, @private_key_obj, 'RS256', @headers)
63
- Log.log.debug{"assertion=[#{assertion}]"}
64
62
  return create_token_call(base_params.merge(grant_type: GRANT_TYPE, assertion: assertion))
65
63
  end
66
64
  end
@@ -11,8 +11,8 @@ module Aspera
11
11
  class << self
12
12
  attr_accessor :additional_info
13
13
  end
14
- # @param redirect_uri url to receive the code after auth (to be exchanged for token)
15
- # @param path_authorize path to login page on web app
14
+ # @param redirect_uri [String] url to receive the code after auth (to be exchanged for token)
15
+ # @param path_authorize [String] path to login page on web app
16
16
  def initialize(
17
17
  redirect_uri:,
18
18
  path_authorize: 'authorize',
@@ -22,9 +22,9 @@ module Aspera
22
22
  @redirect_uri = redirect_uri
23
23
  @path_authorize = path_authorize
24
24
  uri = URI.parse(@redirect_uri)
25
- Aspera.assert_values(uri.scheme, %w[http https]){'redirect_uri scheme must be http or https'}
25
+ Aspera.assert_values(uri.scheme, %w[http https]) { 'redirect_uri scheme must be http or https' }
26
26
  Aspera.assert(!uri.port.nil?, 'redirect_uri must have a port')
27
- # TODO: we could check that host is localhost or local address, as we are going to listen locally
27
+ Aspera.assert(%w[127.0.0.1 ::1 localhost].include?(uri.host)) { 'redirect_uri host must be localhost or loopback address' }
28
28
  end
29
29
 
30
30
  def create_token
@@ -35,14 +35,14 @@ module Aspera
35
35
  base_params.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state)
36
36
  )
37
37
  # here, we need a human to authorize on a web page
38
- Log.log.debug{"login_page_url=#{login_page_url}"}
38
+ Log.log.debug { "login_page_url=#{login_page_url}" }
39
39
  # start a web server to receive request code
40
40
  web_server = WebAuth.new(@redirect_uri, self.class.additional_info)
41
41
  # start browser on login page
42
42
  Environment.instance.open_uri(login_page_url)
43
43
  # wait for code in request
44
44
  received_params = web_server.received_request
45
- Aspera.assert(random_state.eql?(received_params['state'])){'wrong received state'}
45
+ Aspera.assert(random_state.eql?(received_params['state'])) { 'wrong received state' }
46
46
  # exchange code for token
47
47
  return create_token_call(base_params(add_secret: true).merge(
48
48
  grant_type: 'authorization_code',
@@ -8,21 +8,21 @@ module Aspera
8
8
  # Persist data on file system
9
9
  class PersistencyActionOnce
10
10
  DELETE_DEFAULT = lambda(&:empty?)
11
- PARSE_DEFAULT = lambda{ |t| JSON.parse(t)}
12
- FORMAT_DEFAULT = lambda{ |h| JSON.generate(h)}
13
- MERGE_DEFAULT = lambda{ |current, file| current.concat(file).uniq rescue current}
11
+ PARSE_DEFAULT = lambda { |t| JSON.parse(t) }
12
+ FORMAT_DEFAULT = lambda { |h| JSON.generate(h) }
13
+ MERGE_DEFAULT = lambda { |current, file| current.concat(file).uniq rescue current }
14
14
  MANAGER_METHODS = %i[get put delete]
15
15
  private_constant :DELETE_DEFAULT, :PARSE_DEFAULT, :FORMAT_DEFAULT, :MERGE_DEFAULT, :MANAGER_METHODS
16
16
 
17
- # @param :manager Mandatory Database
18
- # @param :data Mandatory object to persist, must be same object from begin to end (assume array by default)
19
- # @param :id Mandatory identifiers
20
- # @param :delete Optional delete persistency condition
21
- # @param :parse Optional parse method (default to JSON)
22
- # @param :format Optional dump method (default to JSON)
23
- # @param :merge Optional merge data from file to current data
17
+ # @param manager [Object] Mandatory Database (must respond to get/put/delete)
18
+ # @param data [Object] Mandatory object to persist, must be same object from begin to end (assume array by default)
19
+ # @param id [String] Mandatory identifier
20
+ # @param delete [Proc] Optional delete persistency condition
21
+ # @param parse [Proc] Optional parse method (default to JSON)
22
+ # @param format [Proc] Optional dump method (default to JSON)
23
+ # @param merge [Proc] Optional merge data from file to current data
24
24
  def initialize(manager:, data:, id:, delete: DELETE_DEFAULT, parse: PARSE_DEFAULT, format: FORMAT_DEFAULT, merge: MERGE_DEFAULT)
25
- Aspera.assert(MANAGER_METHODS.all?{ |i| manager.respond_to?(i)}){"Manager must answer to #{MANAGER_METHODS}"}
25
+ Aspera.assert(MANAGER_METHODS.all? { |i| manager.respond_to?(i) }) { "Manager must answer to #{MANAGER_METHODS}" }
26
26
  Aspera.assert(!data.nil?, 'data must not be nil')
27
27
  Aspera.assert_type(id, String)
28
28
  Aspera.assert(!id.empty?, 'id must not be empty')
@@ -48,7 +48,7 @@ module Aspera
48
48
  end
49
49
  end
50
50
 
51
- # @return internal persisted object, in order to modify its content
51
+ # @return [Array, Hash] internal persisted object, in order to modify its content
52
52
  def data
53
53
  return @persisted_object
54
54
  end
@@ -15,18 +15,19 @@ module Aspera
15
15
  def initialize(folder)
16
16
  @cache = {}
17
17
  @folder = folder
18
- Log.log.debug{"persistency folder: #{@folder}"}
18
+ Log.log.debug { "persistency folder: #{@folder}" }
19
19
  end
20
20
 
21
21
  # Get value of persisted item
22
- # @return [String,nil] Value of persisted id
22
+ # @param object_id [String] Identifier of persisted item
23
+ # @return [String, nil] Persisted value, or nil if not found
23
24
  def get(object_id)
24
- Log.log.debug{"persistency get: #{object_id}"}
25
+ Log.log.debug { "persistency get: #{object_id}" }
25
26
  if @cache.key?(object_id)
26
27
  Log.log.debug('got from memory cache')
27
28
  else
28
29
  persist_filepath = id_to_filepath(object_id)
29
- Log.log.debug{"persistency = #{persist_filepath}"}
30
+ Log.log.debug { "persistency = #{persist_filepath}" }
30
31
  if File.exist?(persist_filepath)
31
32
  Log.log.debug('got from file cache')
32
33
  @cache[object_id] = File.read(persist_filepath)
@@ -42,7 +43,7 @@ module Aspera
42
43
  def put(object_id, value)
43
44
  Aspera.assert_type(value, String)
44
45
  persist_filepath = id_to_filepath(object_id)
45
- Log.log.debug{"persistency saving: #{persist_filepath}"}
46
+ Log.log.debug { "persistency saving: #{persist_filepath}" }
46
47
  FileUtils.rm_f(persist_filepath)
47
48
  File.write(persist_filepath, value)
48
49
  Environment.restrict_file_access(persist_filepath)
@@ -54,37 +55,43 @@ module Aspera
54
55
  # @param object_id [String] Identifier of persisted item
55
56
  def delete(object_id)
56
57
  persist_filepath = id_to_filepath(object_id)
57
- Log.log.debug{"persistency deleting: #{persist_filepath}"}
58
+ Log.log.debug { "persistency deleting: #{persist_filepath}" }
58
59
  FileUtils.rm_f(persist_filepath)
59
60
  @cache.delete(object_id)
60
61
  end
61
62
 
62
- # Delete persisted items
63
+ # Delete persisted items matching the given category, optionally older than a given age
64
+ # @param persist_category [String] Prefix used to identify files belonging to the category
65
+ # @param max_age_seconds [Integer, nil] If set, only delete files older than this many seconds
66
+ # @return [Array<String>] List of deleted file paths
63
67
  def garbage_collect(persist_category, max_age_seconds = nil)
64
68
  garbage_files = current_files(persist_category)
65
69
  if !max_age_seconds.nil?
66
70
  current_time = Time.now
67
- garbage_files.select!{ |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
71
+ garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds }
68
72
  end
69
73
  garbage_files.each do |filepath|
70
74
  File.delete(filepath)
71
- Log.log.debug{"persistency deleted expired: #{filepath}"}
75
+ Log.log.debug { "persistency deleted expired: #{filepath}" }
72
76
  end
73
77
  @cache.clear
74
78
  return garbage_files
75
79
  end
76
80
 
81
+ # List all persisted files matching the given category prefix
82
+ # @param persist_category [String] Prefix identifying the category
83
+ # @return [Array<String>] List of matching file paths
77
84
  def current_files(persist_category)
78
85
  Dir[File.join(@folder, "#{persist_category}*#{FILE_SUFFIX}")]
79
86
  end
80
87
 
81
88
  def current_items(persist_category)
82
- current_files(persist_category).to_h{ |i| [File.basename(i, FILE_SUFFIX), File.read(i)]}
89
+ current_files(persist_category).to_h { |i| [File.basename(i, FILE_SUFFIX), File.read(i)] }
83
90
  end
84
91
 
85
92
  private
86
93
 
87
- # @param object_id String or Array
94
+ # @param object_id [String, Array] identifier or list of identifiers
88
95
  def id_to_filepath(object_id)
89
96
  Aspera.assert_type(object_id, String)
90
97
  FileUtils.mkdir_p(@folder)
@@ -78,18 +78,18 @@ module Aspera
78
78
 
79
79
  # @param filepath [String] Full path to file
80
80
  # @param mimetype [String] MIME typre provided by node API
81
- # @return file type, one of enum CONVERSION_TYPES
81
+ # @return [Symbol] file type, one of enum CONVERSION_TYPES
82
82
  # @raise [RuntimeError] if no conversion type found
83
83
  def conversion_type(filepath, mimetype)
84
- Log.log.debug{"conversion_type(#{filepath},mime=#{mimetype},magic=#{@use_mimemagic})"}
84
+ Log.log.debug { "conversion_type(#{filepath},mime=#{mimetype},magic=#{@use_mimemagic})" }
85
85
  # Default type or empty means no type
86
86
  mimetype = TYPE_NOT_FOUND if mimetype.nil? || (mimetype.is_a?(String) && mimetype.empty?)
87
87
  mimetype = Marcel::MimeType.for(Pathname.new(filepath), name: File.basename(filepath), declared_type: mimetype)
88
88
  mimetype = 'text/plain' if mimetype.eql?(TYPE_NOT_FOUND) && ascii_text_file?(filepath)
89
- raise "no MIME type found for #{File.basename(filepath)}" if mimetype.eql?(TYPE_NOT_FOUND)
89
+ Aspera.assert(!mimetype.eql?(TYPE_NOT_FOUND)) { "no MIME type found for #{File.basename(filepath)}" }
90
90
  conversion_type = mime_to_type(mimetype)
91
- raise "no conversion type found for #{File.basename(filepath)}" if conversion_type.nil?
92
- Log.log.trace1{"conversion_type(#{File.basename(filepath)}): #{conversion_type.class.name} [#{conversion_type}]"}
91
+ Aspera.assert(!conversion_type.nil?) { "no conversion type found for #{File.basename(filepath)}" }
92
+ Log.log.trace1 { "conversion_type(#{File.basename(filepath)}): #{conversion_type.class.name} [#{conversion_type}]" }
93
93
  return conversion_type
94
94
  end
95
95
 
@@ -27,10 +27,10 @@ module Aspera
27
27
  # Node API MIME types are from: http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types.
28
28
  # The resulting preview file type is taken from destination file extension.
29
29
  # Conversion methods are provided by private methods: convert_<conversion_type>_to_<preview_format>.
30
- # -> conversion_type is one of FileTypes::CONVERSION_TYPES.
31
- # -> preview_format is one of Generator::PREVIEW_FORMATS.
30
+ # conversion_type: one of FileTypes::CONVERSION_TYPES.
31
+ # preview_format: one of Generator::PREVIEW_FORMATS.
32
32
  # The conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>.
33
- # -> conversion method is one of Generator::VIDEO_CONVERSION_METHODS.
33
+ # conversion method: one of Generator::VIDEO_CONVERSION_METHODS.
34
34
  # @param src [String] Source file path.
35
35
  # @param dst [String] Destination file path.
36
36
  # @param options [Options] All conversion options.
@@ -57,21 +57,21 @@ module Aspera
57
57
  end
58
58
  end
59
59
  @processing_method = @processing_method.to_sym
60
- Log.log.debug{"method: #{@processing_method}"}
61
- Aspera.assert(respond_to?(@processing_method, true)){"no processing known for #{conversion_type} -> #{@preview_format_sym}"}
60
+ Log.log.debug { "method: #{@processing_method}" }
61
+ Aspera.assert(respond_to?(@processing_method, true)) { "no processing known for #{conversion_type} -> #{@preview_format_sym}" }
62
62
  command = [:magick] + %w[identify -list font]
63
63
  magick_fonts = Utils.parse_magick_fonts(Utils.execute(*command, mode: :capture).first)
64
- Aspera.assert(magick_fonts[:fonts].any?{ |f| f[:name].eql?(@options.thumb_text_font)}){"Missing font #{@options.thumb_text_font} in #{command}"}
64
+ Aspera.assert(magick_fonts[:fonts].any? { |f| f[:name].eql?(@options.thumb_text_font) }) { "Missing font #{@options.thumb_text_font} in #{command}" }
65
65
  end
66
66
 
67
67
  # Creates preview as specified in constructor.
68
68
  def generate
69
- Log.log.debug{"#{@source}->#{@destination} (#{@processing_method})"}
69
+ Log.log.debug { "#{@source}->#{@destination} (#{@processing_method})" }
70
70
  begin
71
71
  send(@processing_method)
72
72
  # Check that generated size does not exceed maximum.
73
73
  result_size = File.size(@destination)
74
- Log.log.warn{"preview size exceeds maximum allowed #{result_size} > #{@options.max_size}"} if result_size > @options.max_size
74
+ Log.log.warn { "preview size exceeds maximum allowed #{result_size} > #{@options.max_size}" } if result_size > @options.max_size
75
75
  ensure
76
76
  FileUtils.rm_rf(@temp_folder)
77
77
  end
@@ -99,7 +99,7 @@ module Aspera
99
99
  # @param index [Integer] Index of part (starts at 1).
100
100
  # @return [Float] Offset in seconds suitable for ffmpeg -ss option.
101
101
  def get_offset(duration, start_offset, total_count, index)
102
- Aspera.assert_type(duration, Float){'duration'}
102
+ Aspera.assert_type(duration, Float) { 'duration' }
103
103
  return start_offset + ((index - 1) * (duration - start_offset) / total_count)
104
104
  end
105
105
 
@@ -127,10 +127,12 @@ module Aspera
127
127
  current_index = last_keyframe + 1 + @options.blend_transframes
128
128
  end
129
129
  Utils.ffmpeg(
130
- in_f: Utils.ffmpeg_fmt(this_tmpdir),
131
- in_p: ['-framerate', @options.blend_fps],
132
- out_f: @destination,
133
- out_p: [
130
+ in: [
131
+ Utils.ffmpeg_fmt(this_tmpdir),
132
+ '-framerate', @options.blend_fps
133
+ ],
134
+ out: [
135
+ @destination,
134
136
  '-filter:v', "scale='trunc(iw/2)*2:trunc(ih/2)*2'",
135
137
  '-codec:v', 'libx264',
136
138
  '-r', frame_rate_hz,
@@ -149,13 +151,15 @@ module Aspera
149
151
  offset_seconds = get_offset(p_duration, @options.video_start_sec.to_i, @options.clips_count.to_i, i)
150
152
  tmp_file_name = format('clip%04d.mp4', i)
151
153
  Utils.ffmpeg(
152
- in_f: @source,
153
- in_p: ['-ss', offset_seconds * 0.9],
154
- out_f: File.join(this_tmpdir, tmp_file_name),
155
- out_p: [
154
+ in: [
155
+ @source,
156
+ '-ss', offset_seconds * 0.9
157
+ ],
158
+ out: [
159
+ File.join(this_tmpdir, tmp_file_name),
156
160
  '-ss', offset_seconds * 0.1,
157
161
  '-t', @options.clips_length,
158
- '-filter:v', "scale=#{@options.video_scale}",
162
+ '-filter:v', "scale='#{@options.video_scale}'",
159
163
  '-codec:a', 'libmp3lame'
160
164
  ]
161
165
  )
@@ -164,10 +168,8 @@ module Aspera
164
168
  end
165
169
  # Concat clips.
166
170
  Utils.ffmpeg(
167
- in_f: file_list_file,
168
- in_p: ['-f', 'concat'],
169
- out_f: @destination,
170
- out_p: ['-codec', 'copy']
171
+ in: [file_list_file, '-f', 'concat'],
172
+ out: [@destination, '-codec', 'copy']
171
173
  )
172
174
  File.delete(file_list_file)
173
175
  end
@@ -176,31 +178,32 @@ module Aspera
176
178
  # Performs a simple re-encoding with configurable ffmpeg options.
177
179
  def convert_video_to_mp4_using_reencode
178
180
  options = @options.reencode_ffmpeg
179
- Aspera.assert_type(options, Hash){'reencode_ffmpeg'}
181
+ Aspera.assert_type(options, Hash) { 'reencode_ffmpeg' }
180
182
  options.each do |k, v|
181
- Aspera.assert_values(k, FFMPEG_OPTIONS_LIST){'key'}
182
- Aspera.assert_type(v, Array){k}
183
+ Aspera.assert_values(k, FFMPEG_OPTIONS_LIST) { 'key' }
184
+ Aspera.assert_type(v, Array) { k }
183
185
  end
186
+ codec = @options.video_codec || Utils.available_h264_encoder
187
+ in_opts = options['in'] || ['-ss', @options.video_start_sec.to_i * 0.9]
188
+ out_opts = options['out'] || [
189
+ '-t', 60,
190
+ '-codec:v', codec,
191
+ '-profile:v', 'high',
192
+ '-pix_fmt', 'yuv420p',
193
+ '-preset', 'slow',
194
+ '-b:v', '500k',
195
+ '-maxrate', '500k',
196
+ '-bufsize', '1000k',
197
+ '-filter:v', "scale='#{@options.video_scale}'",
198
+ '-threads', '0',
199
+ '-codec:a', 'libmp3lame',
200
+ '-ac', '2',
201
+ '-b:a', '128k',
202
+ '-movflags', 'faststart'
203
+ ]
184
204
  Utils.ffmpeg(
185
- in_f: @source,
186
- in_p: options['in'] || ['-ss', @options.video_start_sec.to_i * 0.9],
187
- out_f: @destination,
188
- out_p: options['out'] || [
189
- '-t', 60,
190
- '-codec:v', 'libx264',
191
- '-profile:v', 'high',
192
- '-pix_fmt', 'yuv420p',
193
- '-preset', 'slow',
194
- '-b:v', '500k',
195
- '-maxrate', '500k',
196
- '-bufsize', '1000k',
197
- '-filter:v', "scale=#{@options.video_scale}",
198
- '-threads', '0',
199
- '-codec:a', 'libmp3lame',
200
- '-ac', '2',
201
- '-b:a', '128k',
202
- '-movflags', 'faststart'
203
- ]
205
+ in: [@source, *in_opts],
206
+ out: [@destination, *out_opts]
204
207
  )
205
208
  end
206
209
 
@@ -228,13 +231,13 @@ module Aspera
228
231
  p_max_duration = p_duration
229
232
  end
230
233
  Utils.ffmpeg(
231
- in_f: @source,
232
- in_p: [
234
+ in: [
235
+ @source,
233
236
  '-ss', p_start_offset,
234
237
  '-t', p_max_duration
235
238
  ],
236
- out_f: @destination,
237
- out_p: [
239
+ out: [
240
+ @destination,
238
241
  '-vf', 'fps=5,scale=120:-1:flags=lanczos',
239
242
  '-plays', 0, # Loop forever (0 = infinite loop for APNG).
240
243
  '-f', 'apng'
@@ -265,7 +268,7 @@ module Aspera
265
268
  # soffice creates the file with the source name, so we need to rename it if needed.
266
269
  generated_pdf = File.join(File.dirname(tmp_pdf_file), "#{File.basename(@source, File.extname(@source))}.pdf")
267
270
  FileUtils.mv(generated_pdf, tmp_pdf_file) if generated_pdf != tmp_pdf_file
268
- else Aspera.error_unexpected_value(@options.office_conversion){'office_conversion'}
271
+ else Aspera.error_unexpected_value(@options.office_conversion) { 'office_conversion' }
269
272
  end
270
273
  convert_pdf_to_png(tmp_pdf_file)
271
274
  end
@@ -18,15 +18,16 @@ module Aspera
18
18
  # -x : keep aspect ratio, having value a multiple of x
19
19
  DESCRIPTIONS = [
20
20
  {name: :max_size, default: 1 << 24, description: 'maximum size (in bytes) of preview file'},
21
- {name: :thumb_vid_scale, default: "-1:'min(ih,100)'", description: 'png: video: size (ffmpeg scale argument)'},
21
+ {name: :thumb_vid_scale, default: '-1:min(ih,100)', description: 'png: video: size (ffmpeg scale argument)'},
22
22
  {name: :thumb_vid_fraction, default: 0.1, description: 'png: video: time percent position of snapshot'},
23
23
  {name: :thumb_img_size, default: 800, description: 'png: non-video: height (and width)'},
24
24
  {name: :thumb_text_font, default: 'Courier', description: 'png: plaintext: font for text rendering: `magick identify -list font`'},
25
25
  {name: :office_conversion, default: :soffice, description: 'office: method for office document conversion', values: OFFICE_CONVERSION_METHODS},
26
26
  {name: :video_conversion, default: :reencode, description: 'mp4: method for preview generation', values: VIDEO_CONVERSION_METHODS},
27
27
  {name: :video_png_conv, default: :fixed, description: 'mp4: method for thumbnail generation', values: VIDEO_THUMBNAIL_METHODS},
28
- {name: :video_scale, default: "'min(iw,360)':-2", description: 'mp4: all: video scale (ffmpeg scale argument)'},
28
+ {name: :video_scale, default: 'min(iw,360):-2', description: 'mp4: all: video scale (ffmpeg scale argument)'},
29
29
  {name: :video_start_sec, default: 10, description: 'mp4: all: start offset (seconds) of video preview'},
30
+ {name: :video_codec, default: nil, description: 'mp4: reencode: video codec for ffmpeg (e.g. libx264, h264_videotoolbox, h264_nvenc), default: auto-detect'},
30
31
  {name: :reencode_ffmpeg, default: {}, description: 'mp4: reencode: options to ffmpeg, keys: `in`, `out`'},
31
32
  {name: :blend_keyframes, default: 30, description: 'mp4: blend: # key frames'},
32
33
  {name: :blend_pauseframes, default: 3, description: 'mp4: blend: # pause frames'},