aspera-cli 4.28.0 → 4.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +217 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +122 -26
  10. data/SECURITY.md +84 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1895 -632
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +29 -22
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +95 -67
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +370 -0
  41. data/lib/aspera/ascp/log_locations.rb +163 -0
  42. data/lib/aspera/ascp/management.rb +16 -8
  43. data/lib/aspera/ascp/session_errors.rb +217 -0
  44. data/lib/aspera/assert.rb +4 -0
  45. data/lib/aspera/cli/ascp_actions.rb +12 -19
  46. data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
  47. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  48. data/lib/aspera/cli/bootstrapper.rb +38 -33
  49. data/lib/aspera/cli/command_line.rb +63 -8
  50. data/lib/aspera/cli/command_registry.rb +9 -1
  51. data/lib/aspera/cli/command_spec.rb +47 -7
  52. data/lib/aspera/cli/completion/ascli.bash +19 -5
  53. data/lib/aspera/cli/completion/ascli.fish +13 -2
  54. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  55. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  56. data/lib/aspera/cli/completion_actions.rb +303 -69
  57. data/lib/aspera/cli/context.rb +9 -4
  58. data/lib/aspera/cli/deprecation.rb +5 -2
  59. data/lib/aspera/cli/error.rb +6 -0
  60. data/lib/aspera/cli/extended_value.rb +66 -31
  61. data/lib/aspera/cli/formatter.rb +43 -30
  62. data/lib/aspera/cli/gem_checker.rb +9 -6
  63. data/lib/aspera/cli/hints.rb +22 -10
  64. data/lib/aspera/cli/http.rb +38 -10
  65. data/lib/aspera/cli/info.rb +12 -7
  66. data/lib/aspera/cli/mailer.rb +6 -3
  67. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  68. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  69. data/lib/aspera/cli/mcp/tool.rb +20 -17
  70. data/lib/aspera/cli/option_declarator.rb +10 -4
  71. data/lib/aspera/cli/option_registry.rb +6 -3
  72. data/lib/aspera/cli/option_types.rb +17 -2
  73. data/lib/aspera/cli/option_value.rb +77 -18
  74. data/lib/aspera/cli/options.schema.yaml +641 -80
  75. data/lib/aspera/cli/parser.rb +71 -35
  76. data/lib/aspera/cli/plugins/alee.rb +7 -4
  77. data/lib/aspera/cli/plugins/aoc.rb +248 -37
  78. data/lib/aspera/cli/plugins/ats.rb +10 -9
  79. data/lib/aspera/cli/plugins/base.rb +50 -18
  80. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  81. data/lib/aspera/cli/plugins/config.rb +210 -104
  82. data/lib/aspera/cli/plugins/console.rb +6 -3
  83. data/lib/aspera/cli/plugins/cos.rb +8 -4
  84. data/lib/aspera/cli/plugins/factory.rb +21 -7
  85. data/lib/aspera/cli/plugins/faspex5.rb +31 -26
  86. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  87. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  88. data/lib/aspera/cli/plugins/mcp.rb +42 -22
  89. data/lib/aspera/cli/plugins/node.rb +39 -38
  90. data/lib/aspera/cli/plugins/oauth.rb +23 -6
  91. data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
  92. data/lib/aspera/cli/plugins/preview.rb +72 -45
  93. data/lib/aspera/cli/plugins/server.rb +66 -14
  94. data/lib/aspera/cli/plugins/shares.rb +87 -18
  95. data/lib/aspera/cli/plugins/transferd.rb +228 -0
  96. data/lib/aspera/cli/poll.rb +236 -0
  97. data/lib/aspera/cli/preset_actions.rb +144 -30
  98. data/lib/aspera/cli/preset_manager.rb +113 -35
  99. data/lib/aspera/cli/prompt.rb +157 -9
  100. data/lib/aspera/cli/result.rb +12 -8
  101. data/lib/aspera/cli/runner.rb +89 -48
  102. data/lib/aspera/cli/secret_finder.rb +4 -1
  103. data/lib/aspera/cli/security.rb +89 -9
  104. data/lib/aspera/cli/special_values.rb +4 -1
  105. data/lib/aspera/cli/sync_actions.rb +15 -10
  106. data/lib/aspera/cli/table_fitter.rb +3 -0
  107. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  108. data/lib/aspera/cli/transfer_actions.rb +11 -8
  109. data/lib/aspera/cli/transfer_agent.rb +43 -20
  110. data/lib/aspera/cli/transfer_progress.rb +14 -45
  111. data/lib/aspera/cli/vault_actions.rb +45 -0
  112. data/lib/aspera/cli/vault_manager.rb +188 -42
  113. data/lib/aspera/cli/version.rb +4 -1
  114. data/lib/aspera/cli/web_secret.rb +12 -10
  115. data/lib/aspera/cli/wizard.rb +42 -39
  116. data/lib/aspera/command_line_builder.rb +9 -5
  117. data/lib/aspera/command_line_converter.rb +4 -1
  118. data/lib/aspera/coverage.rb +103 -33
  119. data/lib/aspera/data_repository.rb +4 -1
  120. data/lib/aspera/dot_container.rb +4 -1
  121. data/lib/aspera/environment.rb +23 -12
  122. data/lib/aspera/exec_spec.rb +3 -0
  123. data/lib/aspera/faspex_gw.rb +13 -9
  124. data/lib/aspera/faspex_postproc.rb +15 -12
  125. data/lib/aspera/formatter_interface.rb +4 -1
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +20 -5
  129. data/lib/aspera/inflate.rb +43 -0
  130. data/lib/aspera/json_rpc/client.rb +6 -5
  131. data/lib/aspera/json_rpc/version.rb +3 -0
  132. data/lib/aspera/line_logger.rb +4 -1
  133. data/lib/aspera/link_header.rb +4 -1
  134. data/lib/aspera/log.rb +18 -5
  135. data/lib/aspera/markdown.rb +5 -2
  136. data/lib/aspera/mime.rb +3 -0
  137. data/lib/aspera/nagios.rb +6 -4
  138. data/lib/aspera/node_emulator.rb +15 -13
  139. data/lib/aspera/oauth/client/base.rb +124 -73
  140. data/lib/aspera/oauth/client/boot.rb +7 -4
  141. data/lib/aspera/oauth/client/factory.rb +41 -18
  142. data/lib/aspera/oauth/client/generic.rb +5 -1
  143. data/lib/aspera/oauth/client/json_credentials.rb +7 -3
  144. data/lib/aspera/oauth/client/jwt.rb +6 -3
  145. data/lib/aspera/oauth/client/shares_web.rb +94 -0
  146. data/lib/aspera/oauth/client/url_json.rb +7 -3
  147. data/lib/aspera/oauth/client/web.rb +40 -13
  148. data/lib/aspera/oauth/server/pkce.rb +6 -5
  149. data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +16 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +17 -2
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +3 -0
  165. data/lib/aspera/rest/aspera_errors.rb +4 -1
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +45 -34
  168. data/lib/aspera/rest/error_analyzer.rb +5 -2
  169. data/lib/aspera/rest/list.rb +8 -4
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +8 -5
  172. data/lib/aspera/rest.rb +9 -6
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  175. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  176. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  177. data/lib/aspera/schema/async_tables.yaml +11 -10
  178. data/lib/aspera/schema/documentation.rb +8 -2
  179. data/lib/aspera/schema/reader.rb +7 -0
  180. data/lib/aspera/schema/registry.rb +44 -28
  181. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  182. data/lib/aspera/schema/validator.rb +4 -1
  183. data/lib/aspera/sealed_hash.rb +150 -0
  184. data/lib/aspera/secret/hider.rb +213 -0
  185. data/lib/aspera/secret/reference.rb +150 -0
  186. data/lib/aspera/secret/resolver.rb +212 -0
  187. data/lib/aspera/secret/store/base.rb +153 -0
  188. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  189. data/lib/aspera/secret/store/factory.rb +103 -0
  190. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  191. data/lib/aspera/secret/store/keeper.rb +141 -0
  192. data/lib/aspera/secret/store/macos_security.rb +284 -0
  193. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  194. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  195. data/lib/aspera/secret/store/one_password_cli.rb +112 -0
  196. data/lib/aspera/secret/store/secret_service.rb +125 -0
  197. data/lib/aspera/secret/strength.rb +119 -0
  198. data/lib/aspera/servlet_helpers.rb +5 -2
  199. data/lib/aspera/ssh.rb +10 -7
  200. data/lib/aspera/ssl.rb +5 -2
  201. data/lib/aspera/string_ext.rb +3 -0
  202. data/lib/aspera/sync/args.schema.yaml +3 -0
  203. data/lib/aspera/sync/conf.schema.yaml +9 -4
  204. data/lib/aspera/sync/database.rb +3 -0
  205. data/lib/aspera/sync/operations.rb +17 -15
  206. data/lib/aspera/temp_file_manager.rb +8 -4
  207. data/lib/aspera/timer_limiter.rb +3 -0
  208. data/lib/aspera/transfer/error.rb +4 -1
  209. data/lib/aspera/transfer/faux_file.rb +4 -1
  210. data/lib/aspera/transfer/parameters.rb +17 -14
  211. data/lib/aspera/transfer/result.rb +23 -0
  212. data/lib/aspera/transfer/resumer.rb +6 -4
  213. data/lib/aspera/transfer/spec.rb +9 -22
  214. data/lib/aspera/units/decimal.rb +67 -0
  215. data/lib/aspera/units/duration.rb +109 -0
  216. data/lib/aspera/uri_reader.rb +64 -9
  217. data/lib/aspera/web_auth/index.html.erb +6 -2
  218. data/lib/aspera/web_auth/paste.rb +109 -0
  219. data/lib/aspera/web_auth/receiver.rb +50 -0
  220. data/lib/aspera/web_auth/relay.rb +76 -0
  221. data/lib/aspera/web_auth/reply.html.erb +150 -0
  222. data/lib/aspera/web_auth/server.rb +47 -10
  223. data/lib/aspera/web_page.rb +7 -2
  224. data/lib/aspera/web_server_simple.rb +6 -3
  225. data/lib/aspera/yaml.rb +5 -1
  226. data/llms-install.md +88 -0
  227. data/llms.txt +87 -0
  228. data/skills/ascli/SKILL.md +94 -0
  229. data.tar.gz.sig +0 -0
  230. metadata +77 -20
  231. metadata.gz.sig +0 -0
  232. data/lib/aspera/duration.rb +0 -42
  233. data/lib/aspera/keychain/base.rb +0 -30
  234. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  235. data/lib/aspera/keychain/factory.rb +0 -58
  236. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  237. data/lib/aspera/keychain/macos_security.rb +0 -277
  238. data/lib/aspera/keychain/one_password_api.rb +0 -91
  239. data/lib/aspera/keychain/one_password_base.rb +0 -39
  240. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  241. data/lib/aspera/secret_hider.rb +0 -196
  242. data/lib/aspera/secret_strength.rb +0 -114
@@ -0,0 +1,370 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # cspell:ignore asperascp asws getname lastlog lsass pcap tokengen trapd userdata winuser wtmp
7
+ require_relative '../assert'
8
+ require_relative '../log'
9
+ require_relative 'log_analyzer/log_file'
10
+ require_relative 'log_analyzer/records'
11
+ require 'resolv'
12
+ require 'set'
13
+ require 'stringio'
14
+ require 'time'
15
+ require 'zlib'
16
+
17
+ module Aspera
18
+ module Ascp
19
+ # Analysis of ascp transfer logs: sessions, statistics and diagnostics.
20
+ # Results are tables: Hash (one object) or Array of Hash (list of objects).
21
+ class LogAnalyzer
22
+ # Oldest ascp version for which the analysis is designed
23
+ MIN_VERSION = '4.0'
24
+ # First two bytes of a gzip file
25
+ GZIP_MAGIC = "\x1F\x8B".b.freeze
26
+ # Files analyzed when scanning a folder
27
+ INCLUDED_FILES = /transfer|messages|log/i
28
+ # Files not analyzed when scanning a folder (path relative to folder)
29
+ EXCLUDED_FILES = %r{async|ssl_|\.dat|\.xml|\.html|graph|winuser|manager|asperascp\.20|background|asws|node|plugin|opt/aspera/var|\.exe|trapd|rtf|lsass|lastlog|wtmp|
30
+ process|stats|production|mongrel|console|ntp\.|prepost|aspx|aspshell|tokengen|userdata|http|asconfigurator|ascmd|\.jpg|\.png|ui\.|connect|ssh|
31
+ stderr|stdout|central|sync|\.zip|\.msi|\.tgz|\.tbz|\.tar|\.rar|\.bz2?|\.pcap|analysis\.txt|\.7z}ix
32
+ # Folders not scanned
33
+ EXCLUDED_FOLDERS = /\$|recycle/i
34
+ # Statistics summed over log files
35
+ SUMMED_STATS = %i[total_sessions success failed cancelled http_fallback smart_reset win_clock retries] + LogFile::FINDINGS.keys
36
+ # Severities of diagnostics, most severe first
37
+ SEVERITIES = %w[error warning info].freeze
38
+ # Maximum values of log files
39
+ MAXIMA = {max_total: 'max_throughput', max_out: 'max_send', max_in: 'max_receive', max_sess: 'max_sessions'}.freeze
40
+
41
+ # Default diagnostic thresholds
42
+ DEFAULT_THRESHOLDS = {
43
+ efficiency: 90.0,
44
+ control_loss_percent: 5.0,
45
+ control_loss_sent: 10,
46
+ retry_window: 3600,
47
+ last_error_window: 60,
48
+ rate_decline_ratio: 0.7,
49
+ rate_decline_min_samples: 10,
50
+ rate_decline_delay: 20,
51
+ rate_decline_end_fraction: 0.1,
52
+ license_expiry_days: 30,
53
+ small_files_min_count: 100,
54
+ small_files_max_mean: 1_000_000,
55
+ small_mtu_max: 1400
56
+ }.freeze
57
+
58
+ # @return [Array<LogFile>] analyzed log files
59
+ attr_reader :log_files, :thresholds
60
+
61
+ # @param period [Integer] Snapshot period in seconds
62
+ # @param zero_bytes [Boolean] Include sessions without transferred bytes
63
+ # @param mtu [Integer, nil] MTU to use when not found in logs
64
+ # @param resolve [Boolean] Resolve peer addresses to host names
65
+ # @param thresholds [Hash] Diagnostic thresholds
66
+ # @param timezone [String, nil] Time zone of the log lines: `UTC` or `+HH:MM`. Time zone of the analyst if nil.
67
+ def initialize(period: 60, zero_bytes: false, mtu: nil, resolve: false, thresholds: {}, timezone: nil)
68
+ self.class.check_timezone(timezone)
69
+ @timezone = timezone
70
+ thresholds = thresholds.transform_keys(&:to_sym)
71
+ unknown = thresholds.keys - DEFAULT_THRESHOLDS.keys
72
+ Aspera.assert(unknown.empty?, type: ArgumentError) { "Unknown threshold: #{unknown.join(', ')}" }
73
+ @thresholds = DEFAULT_THRESHOLDS.merge(thresholds)
74
+ @options = {period: period, zero_bytes: zero_bytes, mtu: mtu, thresholds: @thresholds, timezone: timezone}
75
+ @resolve = resolve
76
+ @log_files = []
77
+ end
78
+
79
+ class << self
80
+ # @param timezone [String, nil] Time zone of the log lines: `UTC` or `+HH:MM`
81
+ # @raise [ParameterError] if the time zone is not valid
82
+ def check_timezone(timezone)
83
+ return if timezone.nil?
84
+ Time.at(0, in: timezone)
85
+ rescue ArgumentError
86
+ raise ParameterError, "Invalid time zone: #{timezone.inspect}, expected `UTC` or `+HH:MM`"
87
+ end
88
+
89
+ # @param path [String] log file, `-` (standard input), or folder
90
+ # @return [Array<String>] the log file, or the log files in the folder (recursively)
91
+ def log_paths(path)
92
+ return [path] unless File.directory?(path)
93
+ without_links(scan_folder(path, path, []))
94
+ end
95
+
96
+ # Read the lines of a log file: plain, or compressed with gzip (detected with its first bytes)
97
+ # @param path [String] Log file, or `-` for the standard input
98
+ # @param encoding [Encoding] Encoding of lines
99
+ # @yieldparam line [String]
100
+ def read_lines(path, encoding, &block)
101
+ io = path == STDIN_PATH ? StringIO.new($stdin.binmode.read) : File.open(path, 'rb')
102
+ gzip = io.read(2) == GZIP_MAGIC
103
+ io.rewind
104
+ io = Zlib::GzipReader.new(io, external_encoding: encoding) if gzip
105
+ io.set_encoding(encoding) unless gzip
106
+ io.each_line(&block)
107
+ ensure
108
+ io&.close
109
+ end
110
+
111
+ # Load the log files of a path
112
+ # In a folder, a file that is not an ascp log is ignored. A file given explicitly must be an ascp log.
113
+ # @param path [String] log file, `-` (standard input), or folder
114
+ # @yieldparam file [String] log file
115
+ # @yieldreturn [#ascp?] the loaded log
116
+ # @return [Array] the loaded logs that are ascp logs
117
+ def load_logs(path)
118
+ explicit = !File.directory?(path)
119
+ log_paths(path).filter_map do |file|
120
+ log = yield(file)
121
+ next log if log.ascp?
122
+ Aspera.assert(!explicit, type: ParameterError) { "Not an ascp log: #{file}" }
123
+ Log.log.debug { "Ignored, not an ascp log: #{file}" }
124
+ nil
125
+ rescue Zlib::Error, Errno::EPIPE => e
126
+ # JRuby raises Errno::EPIPE (instead of Zlib::Error) on truncated gzip input
127
+ raise ParameterError, "Cannot read #{file}: #{e.message}" if explicit
128
+ Log.log.warn("Ignored, cannot read #{file}: #{e.message}")
129
+ nil
130
+ end
131
+ end
132
+
133
+ # Reads the file up to its first `ascp` line only
134
+ # @param path [String] log file, plain or compressed with gzip
135
+ # @return [Boolean] true if the file contains `ascp` log lines
136
+ def ascp_log?(path)
137
+ read_lines(path, Encoding::BINARY) { |line| return true if line.match?(Lines::ASCP_LINE) }
138
+ false
139
+ rescue Zlib::Error, Errno::EPIPE
140
+ false
141
+ end
142
+
143
+ private
144
+
145
+ # `ascp` creates `<name>.log` as a hard link to the current rotated file `<name>.<n>.log`: keep only one of them.
146
+ # @return [Array<String>] files, without those that are the same file as a previous one
147
+ def without_links(files)
148
+ seen = Set.new
149
+ files.select do |file|
150
+ stat = File.stat(file)
151
+ # No inode number on some platforms
152
+ stat.ino.zero? || seen.add?([stat.dev, stat.ino])
153
+ end
154
+ end
155
+
156
+ # @return [Array<String>] files, with the log files of folder added
157
+ def scan_folder(root, folder, files)
158
+ children = begin
159
+ Dir.children(folder)
160
+ rescue SystemCallError => e
161
+ # A sub-folder that cannot be read is skipped, the folder given is not
162
+ raise if folder.eql?(root)
163
+ Log.log.warn { "Ignored, cannot read #{folder}: #{e.message}" }
164
+ []
165
+ end
166
+ children.sort.each do |name|
167
+ next if name.start_with?('.')
168
+ path = File.join(folder, name)
169
+ if File.directory?(path)
170
+ scan_folder(root, path, files) unless name.match?(EXCLUDED_FOLDERS)
171
+ elsif name.match?(INCLUDED_FILES) && !path.delete_prefix(root).match?(EXCLUDED_FILES)
172
+ files.push(path)
173
+ end
174
+ end
175
+ files
176
+ end
177
+ end
178
+
179
+ # Analyze a log file, or the log files in a folder (recursively)
180
+ # @param path [String] file, `-` (standard input) or folder
181
+ # @return [self]
182
+ def add(path)
183
+ @log_files.concat(self.class.load_logs(path) { |file| LogFile.new(file, **@options).analyze })
184
+ self
185
+ end
186
+
187
+ # @return [Hash] statistics over all log files
188
+ def summary
189
+ stats = Hash.new(0)
190
+ errors = Hash.new(0)
191
+ categories = Hash.new(0)
192
+ result = {}
193
+ @log_files.each do |log|
194
+ SUMMED_STATS.each { |key| stats[key] += log.stats[key] }
195
+ stats[:stuck_nack_loss] = [stats[:stuck_nack_loss], log.stats[:stuck_nack_loss]].max
196
+ log.stats[:errors].each { |error, count| errors[error] += count }
197
+ log.stats[:error_categories].each { |category, count| categories[category] += count }
198
+ MAXIMA.each do |key, name|
199
+ next if log.stats[key] <= result.fetch(name, 0)
200
+ result[name] = log.stats[key]
201
+ result["#{name}_time"] = iso(log.stats[:"#{key}_time"])
202
+ end
203
+ end
204
+ first = @log_files.filter_map(&:first_time).min
205
+ last = @log_files.filter_map(&:last_time).max
206
+ {
207
+ 'files' => @log_files.size,
208
+ 'first_log' => iso(first),
209
+ 'last_log' => iso(last),
210
+ 'sessions' => stats[:total_sessions],
211
+ 'success' => stats[:success],
212
+ 'failed' => stats[:failed],
213
+ 'cancelled' => stats[:cancelled],
214
+ 'success_rate' => ((100.0 * (stats[:total_sessions] - stats[:failed]) / stats[:total_sessions]).round(2) if stats[:total_sessions].positive?),
215
+ 'http_fallback' => stats[:http_fallback],
216
+ 'retries' => stats[:retries],
217
+ 'multi_session' => @log_files.sum { |log| log.multi_sessions.size },
218
+ 'errors' => errors.sort_by { |_, count| -count }.to_h,
219
+ 'error_categories' => categories.sort_by { |_, count| -count }.to_h,
220
+ 'diagnostics' => SEVERITIES.to_h { |severity| [severity, 0] }.merge(diagnostics.map { |row| row['severity'] }.tally).reject { |_, count| count.zero? }
221
+ }.merge(result).merge(
222
+ 'findings' => LogFile.findings(stats),
223
+ 'warnings' => warnings
224
+ ).compact
225
+ end
226
+
227
+ # @return [Array<String>] warnings about log files
228
+ def warnings
229
+ @log_files.flat_map do |log|
230
+ log.old_versions.map { |version| "#{log.path}: ascp #{version} is older than #{MIN_VERSION}, analysis may be inaccurate" }
231
+ end
232
+ end
233
+
234
+ # @return [Array<Hash>] one row per session, with the other side if its log is also analyzed
235
+ def sessions
236
+ rows = @log_files.flat_map(&:session_reports)
237
+ join_sides(rows)
238
+ single_file(rows)
239
+ end
240
+
241
+ # @return [Array<Hash>] one row per diagnostic of sessions: id, severity, message, recommendation (if the diagnostic has one)
242
+ def diagnostics
243
+ single_file(@log_files.flat_map(&:diagnostics))
244
+ end
245
+
246
+ # @return [Array<Hash>] one row per multi-session transfer
247
+ def multi_sessions
248
+ single_file(@log_files.flat_map(&:multi_sessions))
249
+ end
250
+
251
+ # @return [Array<Hash>] one row per snapshot period: aggregate rates
252
+ def snapshots
253
+ single_file(@log_files.flat_map(&:snapshots))
254
+ end
255
+
256
+ # @return [Array<Hash>] one row per session in each snapshot period
257
+ def snapshot_sessions
258
+ single_file(@log_files.flat_map(&:snapshot_sessions))
259
+ end
260
+
261
+ # @return [Array<Hash>] sessions and errors by peer address, most failures first
262
+ def peers
263
+ rows = {}
264
+ each_session do |_, s|
265
+ peer = s[:peer] or next
266
+ row = (rows[peer] ||= {'peer' => peer, 'users' => Set.new, 'sessions' => 0, 'success' => 0, 'failed' => 0, 'cancelled' => 0, 'errors' => Hash.new(0)})
267
+ row['users'].add(s[:user]) if s[:user]
268
+ row['sessions'] += 1
269
+ case s[:status]
270
+ when 'success' then row['success'] += 1
271
+ when 'failed'
272
+ row[s[:error].to_s.match?(/cancelled/i) ? 'cancelled' : 'failed'] += 1
273
+ row['errors'][s[:error]] += 1 if s[:error]
274
+ end
275
+ end
276
+ rows.values.sort_by { |row| -row['failed'] }.map do |row|
277
+ row.merge('users' => row['users'].to_a, 'errors' => row['errors'].to_h).merge(hostname(row['peer']))
278
+ end
279
+ end
280
+
281
+ # @return [Array<Hash>] one row per transferred file
282
+ def transfers
283
+ result = []
284
+ each_session do |log, s|
285
+ s.files.each do |name, file|
286
+ result.push({
287
+ 'file' => log.path,
288
+ 'session' => s[:uuid],
289
+ 'pid' => s.pid,
290
+ 'user' => s[:user],
291
+ 'peer' => s[:peer],
292
+ 'destination' => s[:dest],
293
+ 'start' => iso(s[:start]),
294
+ 'stop' => iso(s[:stop]),
295
+ 'path' => name,
296
+ 'direction' => file[:op] == 'recv' ? 'receive' : file[:op],
297
+ 'size' => file[:size],
298
+ 'status' => file[:status],
299
+ 'error' => file[:error]
300
+ }.compact)
301
+ end
302
+ end
303
+ single_file(result)
304
+ end
305
+
306
+ # @return [Array<Hash>] licenses of local and peer systems
307
+ def licenses
308
+ @log_files.each_with_object({}) do |log, all|
309
+ log.licenses.each { |key, license| all[key] = license.merge(file: log.path) }
310
+ end.map do |(number, address), license|
311
+ {'license' => number, 'address' => address, 'product' => license[:product], 'account' => license[:account], 'os' => license[:os], 'file' => license[:file]}.compact
312
+ end
313
+ end
314
+
315
+ # @return [Array<Hash>] by peer: number of sessions with loss due to overdriving of network (shallow queues) or storage
316
+ def overdrives
317
+ counts = Hash.new(0)
318
+ each_session do |_, s|
319
+ cause = s.overdrive
320
+ counts[[s[:peer], cause]] += 1 if cause
321
+ end
322
+ counts.sort_by { |_, count| -count }.map do |(peer, cause), count|
323
+ {'peer' => peer, 'cause' => cause.to_s, 'occurrences' => count}.merge(hostname(peer))
324
+ end
325
+ end
326
+
327
+ private
328
+
329
+ def iso(time)
330
+ Time.at(time).getlocal(@timezone).iso8601 if time
331
+ end
332
+
333
+ def each_session
334
+ @log_files.each { |log| log.sessions.each { |s| yield(log, s) } }
335
+ end
336
+
337
+ # Same session in the logs of sender and receiver: add the other side to each row
338
+ # @param rows [Array<Hash>] sessions
339
+ def join_sides(rows)
340
+ rows.group_by { |row| row['session'] }.each do |uuid, same|
341
+ next if uuid.nil? || same.size != 2 || same.map { |row| row['direction'] }.uniq.size != 2
342
+ same.each_with_index do |row, index|
343
+ other = same[1 - index]
344
+ row['peer_side'] = other.slice('file', 'pid', 'role', 'status', 'error').compact
345
+ row['peer_diagnostics'] = other['diagnostics'] if other['diagnostics']&.any?
346
+ end
347
+ end
348
+ end
349
+
350
+ # Remove column `file` when only one file is analyzed
351
+ def single_file(rows)
352
+ return rows if @log_files.size > 1
353
+ rows.each { |row| row.delete('file') }
354
+ end
355
+
356
+ # @return [Hash] host name of address, if resolution is enabled and successful
357
+ def hostname(address)
358
+ return {} unless @resolve && address
359
+ @hostnames ||= {}
360
+ @hostnames[address] ||= Resolv::DNS.open do |dns|
361
+ dns.timeouts = 5
362
+ dns.getname(address).to_s
363
+ rescue Resolv::ResolvError, ArgumentError
364
+ ''
365
+ end
366
+ @hostnames[address].empty? ? {} : {'hostname' => @hostnames[address]}
367
+ end
368
+ end
369
+ end
370
+ end
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../environment'
7
+ require_relative '../log'
8
+ require_relative '../products/desktop'
9
+ require_relative 'log_analyzer'
10
+ require 'json'
11
+ require 'set'
12
+ require 'xmlsimple'
13
+
14
+ module Aspera
15
+ module Ascp
16
+ # Location of the `ascp` log files of the local system.
17
+ # `ascp` logs in the folder of option `-L` (set by Connect, transferd, IBM Aspera for Desktop),
18
+ # else in `<logging><directory>` of `aspera.conf`, else in a default location that depends on the OS:
19
+ # syslog on Linux and Unix, `~/Library/Logs/Aspera` on macOS, `var\log` of the product (or folder of `ascp`) on Windows.
20
+ module LogLocations
21
+ # Linux and Unix: `ascp` logs to syslog (facility `local2`), in one of these files depending on the syslog configuration
22
+ SYSLOG_FILES = %w[/var/log/aspera.log /var/log/messages /var/log/syslog].freeze
23
+ # Log location of `ascp` on Linux and Unix without `-L` nor `<logging><directory>`
24
+ SYSLOG = 'syslog'
25
+ # Source of a location that is not specific to a product
26
+ DEFAULT_SOURCE = 'ascp default'
27
+ # Source of a location given by an `ascp` found in `PATH`, but not in the locations of known products
28
+ PATH_SOURCE = 'ascp in PATH'
29
+ # Names of the log files written by `ascp` in a folder (`-L` or `aspera.conf`): current file, rotations, compressed
30
+ ASCP_FILE = /\A(?:aspera-scp-transfer|ascp4?)(?:\.\d+)?\.log(?:\.gz)?\z/
31
+ class << self
32
+ # @param products [Array<Hash>] installed products, see `Installation#installed_products`
33
+ # @param defaults [Array<Array>] default locations: `[path, source]`
34
+ # @return [Array<Hash>] files that contain `ascp` log lines: `path`, `size`, `modified`, `source`, most recent first
35
+ def find(products, defaults: default_locations)
36
+ seen = Set.new
37
+ # Default locations first: they are shared by products (e.g. `~/Library/Logs/Aspera`)
38
+ locations = defaults + (products + path_products(products)).flat_map { |product| product_locations(product) }
39
+ files = locations.flat_map { |location, source| location_files(location).map { |file| [file, source] } }
40
+ files.filter_map do |file, source|
41
+ stat = File.stat(file)
42
+ # Same file in several locations, or `<name>.log` hard link to `<name>.<n>.log`
43
+ next unless seen.add?(stat.ino.zero? ? File.expand_path(file) : [stat.dev, stat.ino])
44
+ next unless LogAnalyzer.ascp_log?(file)
45
+ {'path' => file, 'size' => stat.size, 'modified' => stat.mtime, 'source' => source}
46
+ end.sort_by { |row| -row['modified'].to_f }
47
+ end
48
+
49
+ # @return [Array<Array>] locations not specific to an installed product: `[path, source]`
50
+ def default_locations
51
+ locations = case Environment.instance.os
52
+ when Environment::OS_WINDOWS then []
53
+ when Environment::OS_MACOS then [File.join(Dir.home, 'Library', 'Logs', 'Aspera'), File.join('', 'Library', 'Logs', 'Aspera')]
54
+ else SYSLOG_FILES
55
+ end.map { |path| [path, DEFAULT_SOURCE] }
56
+ desktop = Products::Desktop.log_folder
57
+ locations.push([desktop, Products::Desktop::APP_NAME]) unless desktop.nil?
58
+ service = transferd_service_log_folder
59
+ locations.push([service, 'transferd service']) unless service.nil?
60
+ locations
61
+ end
62
+
63
+ # Location of the log of `ascp` when it runs without option `-L`
64
+ # @param ascp [String] path of `ascp`
65
+ # @param root [String, nil] root folder of the product, as reported by `ascp` (Windows)
66
+ # @param conf_file [String, nil] `aspera.conf` used by `ascp`
67
+ # @return [String] log folder, or `syslog` (Linux and Unix)
68
+ def ascp_log_location(ascp, root: nil, conf_file: nil)
69
+ folder = conf_directory(conf_file)
70
+ return folder unless folder.nil?
71
+ case Environment.instance.os
72
+ when Environment::OS_MACOS
73
+ File.join(Dir.home.start_with?('/var/root') ? '' : Dir.home, 'Library', 'Logs', 'Aspera')
74
+ when Environment::OS_WINDOWS
75
+ var_log = File.join(root || File.dirname(File.dirname(ascp)), 'var', 'log')
76
+ File.directory?(var_log) ? var_log : File.dirname(ascp)
77
+ else SYSLOG
78
+ end
79
+ end
80
+
81
+ private
82
+
83
+ # @param products [Array<Hash>] installed products
84
+ # @return [Array<Hash>] products for the `ascp` found in `PATH` that are not one of `products`
85
+ def path_products(products)
86
+ known = products.filter_map { |product| real_path(product[:ascp_path]) }
87
+ ENV.fetch('PATH', '').split(File::PATH_SEPARATOR).filter_map do |folder|
88
+ ascp = real_path(File.join(folder, Environment.instance.exe_file('ascp')))
89
+ next if ascp.nil? || known.include?(ascp)
90
+ known.push(ascp)
91
+ bin = File.dirname(ascp)
92
+ {name: PATH_SOURCE, ascp_path: ascp, app_root: File.basename(bin).eql?('bin') ? File.dirname(bin) : bin}
93
+ end
94
+ end
95
+
96
+ # @param product [Hash] installed product
97
+ # @return [Array<Array>] locations of logs of the product: `[path, source]`
98
+ def product_locations(product)
99
+ locations = [product[:log_root], conf_log_folder(product), File.join(product[:app_root], 'var', 'log')]
100
+ # Windows: without `var\log`, `ascp` logs in its own folder
101
+ locations.push(File.dirname(product[:ascp_path])) if Environment.instance.os.eql?(Environment::OS_WINDOWS)
102
+ locations.compact.uniq.map { |path| [path, product[:name]] }
103
+ end
104
+
105
+ # @param product [Hash] installed product
106
+ # @return [String, nil] `<default><logging><directory>` of `aspera.conf` of the product: in `etc`, or in the folder of `ascp` (SDK)
107
+ def conf_log_folder(product)
108
+ conf_directory([File.join(product[:app_root], 'etc', 'aspera.conf'), File.join(File.dirname(product[:ascp_path]), 'aspera.conf')].find { |f| File.file?(f) })
109
+ end
110
+
111
+ # @param conf_file [String, nil] `aspera.conf`
112
+ # @return [String, nil] `<default><logging><directory>` of the file
113
+ def conf_directory(conf_file)
114
+ return if conf_file.nil? || !File.file?(conf_file)
115
+ folder = XmlSimple.xml_in(File.read(conf_file), {'ForceArray' => false}).dig('default', 'logging', 'directory')
116
+ folder if folder.is_a?(String) && !folder.empty?
117
+ rescue StandardError => e
118
+ Log.log.debug { "Cannot read #{conf_file}: #{e.message}" }
119
+ nil
120
+ end
121
+
122
+ # @return [String, nil] `fasp_runtime.log.dir` (folder given to `ascp -L`) of the transferd service installed by `transferd service install`
123
+ def transferd_service_log_folder
124
+ require_relative '../products/transferd_service'
125
+ file = Products::TransferdService::CONFIG_FILE
126
+ return unless File.file?(file) && File.readable?(file)
127
+ folder = JSON.parse(File.read(file)).dig('fasp_runtime', 'log', 'dir')
128
+ folder if folder.is_a?(String) && File.absolute_path?(folder)
129
+ rescue StandardError => e
130
+ Log.log.debug { "Cannot read transferd configuration: #{e.message}" }
131
+ nil
132
+ end
133
+
134
+ # @return [Boolean] true if readable, else warn
135
+ def readable?(location)
136
+ return true if File.readable?(location)
137
+ Log.log.warn { "Cannot read #{location}: permission denied" }
138
+ false
139
+ end
140
+
141
+ # @return [String, nil] real path of the file, nil if it does not exist
142
+ def real_path(file)
143
+ File.realpath(file) if file && File.file?(file)
144
+ end
145
+
146
+ # @param location [String] log folder, or syslog file (with its rotations: `<file>-<date>`, `<file>.<n>`, compressed or not)
147
+ # @return [Array<String>] readable files of the location: `ascp` log files of a folder, or syslog files
148
+ def location_files(location)
149
+ files = if File.directory?(location)
150
+ return [] unless readable?(location)
151
+ LogAnalyzer.log_paths(location).select { |file| File.basename(file).match?(ASCP_FILE) }
152
+ else
153
+ Dir.glob("#{location}*").select { |file| File.file?(file) }.sort
154
+ end
155
+ files.select { |file| readable?(file) }
156
+ rescue SystemCallError => e
157
+ Log.log.warn { "Cannot read #{location}: #{e.message}" }
158
+ []
159
+ end
160
+ end
161
+ end
162
+ end
163
+ end
@@ -1,15 +1,17 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/assert'
4
- require 'aspera/log'
5
- require 'aspera/string_ext'
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ require_relative '../assert'
7
+ require_relative '../log'
8
+ require_relative '../string_ext'
6
9
 
7
10
  module Aspera
8
11
  module Ascp
9
12
  # processing of ascp management port events
10
13
  # Reference: `mgmtmess.c`
11
14
  class Management
12
- # cspell:ignore asmgmt faspmgmt
13
15
  # References:
14
16
  # https://www.ibm.com/support/pages/error-code-reference-tables
15
17
  # `asmgmt_err.h` : codes, mnemonics and messages
@@ -98,7 +100,8 @@ module Aspera
98
100
  69 => {r: true, c: 'MAX_ASCP', m: 'Maximum ascp processes on a node exceeded',
99
101
  a: 'Failed to start session maximum transfer sessions exceeded'}
100
102
  }.freeze
101
- # cspell: disable
103
+ # Mnemonic => error, see ERRORS
104
+ ERRORS_BY_MNEMONIC = ERRORS.values.to_h { |error| [error[:c], error] }.freeze
102
105
  # Same as `as_mgmt_cmd_name_str` (`NOTIFICATION` is twice there too)
103
106
  OPERATIONS = %w[
104
107
  NOP
@@ -283,7 +286,7 @@ module Aspera
283
286
  ].freeze
284
287
  # Management port start message
285
288
  PROT_VERSION = '2'
286
- MGT_HEADER = "FASPMGR #{PROT_VERSION}"
289
+ MGT_HEADER = "FASPMGR #{PROT_VERSION}".freeze
287
290
  # empty line is separator to end event information
288
291
  MGT_FRAME_SEPARATOR = ''
289
292
  # fields description for JSON generation
@@ -307,11 +310,16 @@ module Aspera
307
310
  PATH_ESCAPE = "\e"
308
311
  PATH_ESCAPES = {"\n" => 'n', "\r" => 'r', PATH_ESCAPE => PATH_ESCAPE}.freeze
309
312
 
310
- private_constant :OPERATIONS, :PARAMETERS, :MGT_HEADER, :MGT_FRAME_SEPARATOR, :INTEGER_FIELDS, :BOOLEAN_FIELDS, :BOOLEAN_TRUE, :INTEGER_VALUE,
313
+ private_constant :ERRORS_BY_MNEMONIC, :OPERATIONS, :PARAMETERS, :MGT_HEADER, :MGT_FRAME_SEPARATOR, :INTEGER_FIELDS, :BOOLEAN_FIELDS, :BOOLEAN_TRUE, :INTEGER_VALUE,
311
314
  :PATH_FIELDS, :PATH_ESCAPE, :PATH_ESCAPES
312
- # cspell: enable
313
315
 
314
316
  class << self
317
+ # @param mnemonic [String] mnemonic of error, e.g. `CONNECTION_LOST`
318
+ # @return [Hash, nil] error of management port (see ERRORS), nil if unknown
319
+ def error_by_mnemonic(mnemonic)
320
+ ERRORS_BY_MNEMONIC[mnemonic]
321
+ end
322
+
315
323
  # Translate native event name to snake case
316
324
  def field_native_to_snake(name)
317
325
  case name