aspera-cli 4.28.0 → 4.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +5 -10
  4. data/CHANGELOG.md +217 -2
  5. data/CODE_OF_CONDUCT.md +18 -0
  6. data/CONTRIBUTING.md +135 -56
  7. data/GOVERNANCE.md +30 -0
  8. data/LICENSE +201 -0
  9. data/README.md +122 -26
  10. data/SECURITY.md +84 -0
  11. data/bin/ascli +8 -5
  12. data/bin/asession +9 -6
  13. data/docs/README.md +1895 -632
  14. data/docs/test-mcp-with-ai.md +8 -8
  15. data/lib/aspera/agent/base.rb +23 -2
  16. data/lib/aspera/agent/connect.rb +16 -12
  17. data/lib/aspera/agent/desktop.rb +10 -7
  18. data/lib/aspera/agent/direct.rb +20 -19
  19. data/lib/aspera/agent/factory.rb +6 -3
  20. data/lib/aspera/agent/httpgw.rb +7 -5
  21. data/lib/aspera/agent/node.rb +15 -7
  22. data/lib/aspera/agent/transferd.rb +167 -43
  23. data/lib/aspera/api/alee.rb +4 -1
  24. data/lib/aspera/api/aoc.rb +14 -11
  25. data/lib/aspera/api/ats.rb +4 -2
  26. data/lib/aspera/api/cos_node.rb +7 -4
  27. data/lib/aspera/api/faspex.rb +16 -9
  28. data/lib/aspera/api/httpgw.rb +9 -6
  29. data/lib/aspera/api/node.rb +29 -22
  30. data/lib/aspera/ascmd.rb +8 -5
  31. data/lib/aspera/ascp/installation.rb +95 -67
  32. data/lib/aspera/ascp/log.schema.yaml +1084 -0
  33. data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
  34. data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
  35. data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
  36. data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
  37. data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
  38. data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
  39. data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
  40. data/lib/aspera/ascp/log_analyzer.rb +370 -0
  41. data/lib/aspera/ascp/log_locations.rb +163 -0
  42. data/lib/aspera/ascp/management.rb +16 -8
  43. data/lib/aspera/ascp/session_errors.rb +217 -0
  44. data/lib/aspera/assert.rb +4 -0
  45. data/lib/aspera/cli/ascp_actions.rb +12 -19
  46. data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
  47. data/lib/aspera/cli/async_transfer_store.rb +7 -4
  48. data/lib/aspera/cli/bootstrapper.rb +38 -33
  49. data/lib/aspera/cli/command_line.rb +63 -8
  50. data/lib/aspera/cli/command_registry.rb +9 -1
  51. data/lib/aspera/cli/command_spec.rb +47 -7
  52. data/lib/aspera/cli/completion/ascli.bash +19 -5
  53. data/lib/aspera/cli/completion/ascli.fish +13 -2
  54. data/lib/aspera/cli/completion/ascli.ps1 +31 -9
  55. data/lib/aspera/cli/completion/ascli.zsh +22 -6
  56. data/lib/aspera/cli/completion_actions.rb +303 -69
  57. data/lib/aspera/cli/context.rb +9 -4
  58. data/lib/aspera/cli/deprecation.rb +5 -2
  59. data/lib/aspera/cli/error.rb +6 -0
  60. data/lib/aspera/cli/extended_value.rb +66 -31
  61. data/lib/aspera/cli/formatter.rb +43 -30
  62. data/lib/aspera/cli/gem_checker.rb +9 -6
  63. data/lib/aspera/cli/hints.rb +22 -10
  64. data/lib/aspera/cli/http.rb +38 -10
  65. data/lib/aspera/cli/info.rb +12 -7
  66. data/lib/aspera/cli/mailer.rb +6 -3
  67. data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
  68. data/lib/aspera/cli/mcp/secrets.rb +8 -5
  69. data/lib/aspera/cli/mcp/tool.rb +20 -17
  70. data/lib/aspera/cli/option_declarator.rb +10 -4
  71. data/lib/aspera/cli/option_registry.rb +6 -3
  72. data/lib/aspera/cli/option_types.rb +17 -2
  73. data/lib/aspera/cli/option_value.rb +77 -18
  74. data/lib/aspera/cli/options.schema.yaml +641 -80
  75. data/lib/aspera/cli/parser.rb +71 -35
  76. data/lib/aspera/cli/plugins/alee.rb +7 -4
  77. data/lib/aspera/cli/plugins/aoc.rb +248 -37
  78. data/lib/aspera/cli/plugins/ats.rb +10 -9
  79. data/lib/aspera/cli/plugins/base.rb +50 -18
  80. data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
  81. data/lib/aspera/cli/plugins/config.rb +210 -104
  82. data/lib/aspera/cli/plugins/console.rb +6 -3
  83. data/lib/aspera/cli/plugins/cos.rb +8 -4
  84. data/lib/aspera/cli/plugins/factory.rb +21 -7
  85. data/lib/aspera/cli/plugins/faspex5.rb +31 -26
  86. data/lib/aspera/cli/plugins/faspio.rb +7 -5
  87. data/lib/aspera/cli/plugins/httpgw.rb +7 -4
  88. data/lib/aspera/cli/plugins/mcp.rb +42 -22
  89. data/lib/aspera/cli/plugins/node.rb +39 -38
  90. data/lib/aspera/cli/plugins/oauth.rb +23 -6
  91. data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
  92. data/lib/aspera/cli/plugins/preview.rb +72 -45
  93. data/lib/aspera/cli/plugins/server.rb +66 -14
  94. data/lib/aspera/cli/plugins/shares.rb +87 -18
  95. data/lib/aspera/cli/plugins/transferd.rb +228 -0
  96. data/lib/aspera/cli/poll.rb +236 -0
  97. data/lib/aspera/cli/preset_actions.rb +144 -30
  98. data/lib/aspera/cli/preset_manager.rb +113 -35
  99. data/lib/aspera/cli/prompt.rb +157 -9
  100. data/lib/aspera/cli/result.rb +12 -8
  101. data/lib/aspera/cli/runner.rb +89 -48
  102. data/lib/aspera/cli/secret_finder.rb +4 -1
  103. data/lib/aspera/cli/security.rb +89 -9
  104. data/lib/aspera/cli/special_values.rb +4 -1
  105. data/lib/aspera/cli/sync_actions.rb +15 -10
  106. data/lib/aspera/cli/table_fitter.rb +3 -0
  107. data/lib/aspera/cli/terminal_formatter.rb +9 -6
  108. data/lib/aspera/cli/transfer_actions.rb +11 -8
  109. data/lib/aspera/cli/transfer_agent.rb +43 -20
  110. data/lib/aspera/cli/transfer_progress.rb +14 -45
  111. data/lib/aspera/cli/vault_actions.rb +45 -0
  112. data/lib/aspera/cli/vault_manager.rb +188 -42
  113. data/lib/aspera/cli/version.rb +4 -1
  114. data/lib/aspera/cli/web_secret.rb +12 -10
  115. data/lib/aspera/cli/wizard.rb +42 -39
  116. data/lib/aspera/command_line_builder.rb +9 -5
  117. data/lib/aspera/command_line_converter.rb +4 -1
  118. data/lib/aspera/coverage.rb +103 -33
  119. data/lib/aspera/data_repository.rb +4 -1
  120. data/lib/aspera/dot_container.rb +4 -1
  121. data/lib/aspera/environment.rb +23 -12
  122. data/lib/aspera/exec_spec.rb +3 -0
  123. data/lib/aspera/faspex_gw.rb +13 -9
  124. data/lib/aspera/faspex_postproc.rb +15 -12
  125. data/lib/aspera/formatter_interface.rb +4 -1
  126. data/lib/aspera/graphql.rb +5 -1
  127. data/lib/aspera/hash_ext.rb +3 -0
  128. data/lib/aspera/id_generator.rb +20 -5
  129. data/lib/aspera/inflate.rb +43 -0
  130. data/lib/aspera/json_rpc/client.rb +6 -5
  131. data/lib/aspera/json_rpc/version.rb +3 -0
  132. data/lib/aspera/line_logger.rb +4 -1
  133. data/lib/aspera/link_header.rb +4 -1
  134. data/lib/aspera/log.rb +18 -5
  135. data/lib/aspera/markdown.rb +5 -2
  136. data/lib/aspera/mime.rb +3 -0
  137. data/lib/aspera/nagios.rb +6 -4
  138. data/lib/aspera/node_emulator.rb +15 -13
  139. data/lib/aspera/oauth/client/base.rb +124 -73
  140. data/lib/aspera/oauth/client/boot.rb +7 -4
  141. data/lib/aspera/oauth/client/factory.rb +41 -18
  142. data/lib/aspera/oauth/client/generic.rb +5 -1
  143. data/lib/aspera/oauth/client/json_credentials.rb +7 -3
  144. data/lib/aspera/oauth/client/jwt.rb +6 -3
  145. data/lib/aspera/oauth/client/shares_web.rb +94 -0
  146. data/lib/aspera/oauth/client/url_json.rb +7 -3
  147. data/lib/aspera/oauth/client/web.rb +40 -13
  148. data/lib/aspera/oauth/server/pkce.rb +6 -5
  149. data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
  150. data/lib/aspera/persistency_action_once.rb +5 -3
  151. data/lib/aspera/persistency_folder.rb +6 -3
  152. data/lib/aspera/preview/file_types.rb +6 -3
  153. data/lib/aspera/preview/generator.rb +9 -5
  154. data/lib/aspera/preview/options.schema.yaml +3 -3
  155. data/lib/aspera/preview/terminal.rb +16 -6
  156. data/lib/aspera/preview/utils.rb +6 -4
  157. data/lib/aspera/products/connect.rb +5 -2
  158. data/lib/aspera/products/desktop.rb +17 -2
  159. data/lib/aspera/products/other.rb +4 -2
  160. data/lib/aspera/products/transferd.rb +6 -2
  161. data/lib/aspera/products/transferd_service.rb +610 -0
  162. data/lib/aspera/proxy_auto_config.js +2 -2
  163. data/lib/aspera/proxy_auto_config.rb +5 -2
  164. data/lib/aspera/rainbow.rb +3 -0
  165. data/lib/aspera/rest/aspera_errors.rb +4 -1
  166. data/lib/aspera/rest/call_error.rb +3 -0
  167. data/lib/aspera/rest/client.rb +45 -34
  168. data/lib/aspera/rest/error_analyzer.rb +5 -2
  169. data/lib/aspera/rest/list.rb +8 -4
  170. data/lib/aspera/rest/parameters.rb +3 -0
  171. data/lib/aspera/rest/util.rb +8 -5
  172. data/lib/aspera/rest.rb +9 -6
  173. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
  174. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
  175. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
  176. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
  177. data/lib/aspera/schema/async_tables.yaml +11 -10
  178. data/lib/aspera/schema/documentation.rb +8 -2
  179. data/lib/aspera/schema/reader.rb +7 -0
  180. data/lib/aspera/schema/registry.rb +44 -28
  181. data/lib/aspera/schema/transferd-config.schema.json +333 -0
  182. data/lib/aspera/schema/validator.rb +4 -1
  183. data/lib/aspera/sealed_hash.rb +150 -0
  184. data/lib/aspera/secret/hider.rb +213 -0
  185. data/lib/aspera/secret/reference.rb +150 -0
  186. data/lib/aspera/secret/resolver.rb +212 -0
  187. data/lib/aspera/secret/store/base.rb +153 -0
  188. data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
  189. data/lib/aspera/secret/store/factory.rb +103 -0
  190. data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
  191. data/lib/aspera/secret/store/keeper.rb +141 -0
  192. data/lib/aspera/secret/store/macos_security.rb +284 -0
  193. data/lib/aspera/secret/store/one_password_api.rb +99 -0
  194. data/lib/aspera/secret/store/one_password_base.rb +42 -0
  195. data/lib/aspera/secret/store/one_password_cli.rb +112 -0
  196. data/lib/aspera/secret/store/secret_service.rb +125 -0
  197. data/lib/aspera/secret/strength.rb +119 -0
  198. data/lib/aspera/servlet_helpers.rb +5 -2
  199. data/lib/aspera/ssh.rb +10 -7
  200. data/lib/aspera/ssl.rb +5 -2
  201. data/lib/aspera/string_ext.rb +3 -0
  202. data/lib/aspera/sync/args.schema.yaml +3 -0
  203. data/lib/aspera/sync/conf.schema.yaml +9 -4
  204. data/lib/aspera/sync/database.rb +3 -0
  205. data/lib/aspera/sync/operations.rb +17 -15
  206. data/lib/aspera/temp_file_manager.rb +8 -4
  207. data/lib/aspera/timer_limiter.rb +3 -0
  208. data/lib/aspera/transfer/error.rb +4 -1
  209. data/lib/aspera/transfer/faux_file.rb +4 -1
  210. data/lib/aspera/transfer/parameters.rb +17 -14
  211. data/lib/aspera/transfer/result.rb +23 -0
  212. data/lib/aspera/transfer/resumer.rb +6 -4
  213. data/lib/aspera/transfer/spec.rb +9 -22
  214. data/lib/aspera/units/decimal.rb +67 -0
  215. data/lib/aspera/units/duration.rb +109 -0
  216. data/lib/aspera/uri_reader.rb +64 -9
  217. data/lib/aspera/web_auth/index.html.erb +6 -2
  218. data/lib/aspera/web_auth/paste.rb +109 -0
  219. data/lib/aspera/web_auth/receiver.rb +50 -0
  220. data/lib/aspera/web_auth/relay.rb +76 -0
  221. data/lib/aspera/web_auth/reply.html.erb +150 -0
  222. data/lib/aspera/web_auth/server.rb +47 -10
  223. data/lib/aspera/web_page.rb +7 -2
  224. data/lib/aspera/web_server_simple.rb +6 -3
  225. data/lib/aspera/yaml.rb +5 -1
  226. data/llms-install.md +88 -0
  227. data/llms.txt +87 -0
  228. data/skills/ascli/SKILL.md +94 -0
  229. data.tar.gz.sig +0 -0
  230. metadata +77 -20
  231. metadata.gz.sig +0 -0
  232. data/lib/aspera/duration.rb +0 -42
  233. data/lib/aspera/keychain/base.rb +0 -30
  234. data/lib/aspera/keychain/encrypted_hash.rb +0 -192
  235. data/lib/aspera/keychain/factory.rb +0 -58
  236. data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
  237. data/lib/aspera/keychain/macos_security.rb +0 -277
  238. data/lib/aspera/keychain/one_password_api.rb +0 -91
  239. data/lib/aspera/keychain/one_password_base.rb +0 -39
  240. data/lib/aspera/keychain/one_password_cli.rb +0 -100
  241. data/lib/aspera/secret_hider.rb +0 -196
  242. data/lib/aspera/secret_strength.rb +0 -114
@@ -0,0 +1,97 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # cspell:words nohear overdriven
7
+
8
+ module Aspera
9
+ module Ascp
10
+ class LogAnalyzer
11
+ # Advice shared by several diagnostics
12
+ udp_port = 'Open the FASP UDP port in the firewalls and NAT devices on the path, for both directions, then retry: SSH (TCP) alone is not enough. ' \
13
+ 'The port is 33001 by default (option `-O` of `ascp`, parameter `bind_udp_port` in `transfer/protocol_options` of `aspera.conf` of the server).'
14
+ udp_flow = 'Check that no firewall or NAT device on the path drops the UDP flow (flood protection) or closes it after a timeout, and that the link was not interrupted. ' \
15
+ 'Then retry the transfer, with resume (option `-k` of `ascp`).'
16
+ lower_rate = 'Lower the target rate (option `-l` of `ascp`, or `target_rate` in `bandwidth/flow` of `aspera.conf`) to the capacity of the path, or use fair policy (`--policy=fair`).'
17
+ lower_min = 'Lower the minimum rate (option `-m` of `ascp`, or `min_rate` in `bandwidth/flow` of `aspera.conf`): with fair policy, the minimum rate should stay low.'
18
+ storage_write = 'Check the write speed of the storage of the receiver. Use fair policy, so that storage rate control (`storage_rc/adaptive` in `file_system` of `aspera.conf`) can lower the rate. ' \
19
+ 'Write parameters can be tuned: `write_threads` and `write_block_size` in `file_system` of `aspera.conf`, option `-G` of `ascp`.'
20
+ tcp_connection = 'Check that no firewall or NAT device on the path closes or resets the TCP connection of the SSH session, and the stability of the network.'
21
+ storage_access = 'Check that the storage is available and healthy, has free space, and that the transfer user has the permissions on it, then retry.'
22
+ network_storage = 'Check the availability of the network storage (share, mapped drive) and of its network connection, then retry.'
23
+ # Diagnostic identifier => action to take (English text, like messages of diagnostics).
24
+ # Only for diagnostics on which the user can act. See also `NO_RECOMMENDATION`.
25
+ # Not for `error_<category>` (the message of the diagnostic is already the hint for the category of the error: `SessionErrors::CATEGORIES`).
26
+ RECOMMENDATIONS = {
27
+ client_nohear_server: udp_port,
28
+ server_nohear_client: udp_port,
29
+ timeout_establishing_udp_connection: udp_port,
30
+ http_fallback: "The transfer used HTTP instead of UDP, which is slower (unless forced with option `-yF` of `ascp`). #{udp_port}",
31
+ receiver_data_transfer_timeout: udp_flow,
32
+ inactivity_timer_fired: udp_flow,
33
+ reception_interruption: udp_flow,
34
+ udp_flood_protection: "Check the UDP flood protection of firewalls, and the rate limits of the path. #{lower_rate}",
35
+ low_efficiency: "Retransmissions come from packet loss. #{lower_rate}",
36
+ network_overdriven: "#{lower_rate} Check the path for a device that limits or polices the rate.",
37
+ network_overdrive: "#{lower_rate} Check the path for shallow queues, or a device that limits the rate of UDP.",
38
+ retransmission_buffer_full: "#{lower_rate} Check the path for a device that limits or polices the rate.",
39
+ control_message_loss: "Control messages travel over UDP: their loss shows packet loss on the path. #{lower_rate}",
40
+ feedback_interruption: udp_flow,
41
+ rex_timer_overflow: "The retransmission RTT was excessive: check the path for deep queues, congestion or a clock problem. #{lower_rate}",
42
+ license_rate_limit: 'Use a license with a higher maximum rate on this system (contact your IBM Aspera representative), or set the target rate to the limit of the license ' \
43
+ '(option `-l` of `ascp`, or `target_rate` in `bandwidth/flow` of `aspera.conf`).',
44
+ license_expiry: 'Renew the license of this system before its expiry date (contact your IBM Aspera representative).',
45
+ stuck_nack: "Packet loss prevents the retransmission requests from being served. #{lower_rate}",
46
+ min_rate_high: lower_min,
47
+ min_rate_overdrive: lower_min,
48
+ slow_storage_loss: storage_write,
49
+ slow_disk_write: storage_write,
50
+ disk_buffers: storage_write,
51
+ disk_write_speed_exceeded: storage_write,
52
+ storage_overdrive: storage_write,
53
+ slow_storage_read: 'Check the read speed of the storage of the sender. Read parameters can be tuned: `read_threads` and `read_block_size` in `file_system` of `aspera.conf`, option `-g` of `ascp`.',
54
+ rate_decline: "Check for other traffic on the link, and for slow storage on the receiver. #{lower_rate}",
55
+ small_files: 'Aggregate small files into archives before transfer (`-A zip` of `ascp`), or use a protocol that amortizes the per-file overhead (e.g., HTTP/2 multipart). ' \
56
+ 'For many small files, enable concurrent file opens: `write_threads` in `file_system` of `aspera.conf`.',
57
+ small_mtu: 'Small packets raise the packet rate and the CPU load of ascp. Check whether the path goes through a VPN or tunnel, and raise its MTU; ' \
58
+ 'or check that no device drops the large UDP probes (`ascp` probes 1492 bytes first). To force the datagram size: option `-Z` of `ascp`, or `datagram_size` in `transfer/protocol_options` of `aspera.conf`.',
59
+ reverse_path_skew: 'If the network is not congested, set `rtt_reverse_infer` to `false` in `transfer/protocol_options` of `aspera.conf`: each peer reads its own `aspera.conf`.',
60
+ clock_problem: 'Set `rtt_autocorrect` to `true` in `transfer/protocol_options` of `aspera.conf` (auto-correction of the base RTT, for virtual machines), and check the clock of the virtual machine.',
61
+ peer_clock_problem: 'Ask the administrator of the peer to set `rtt_autocorrect` to `true` in `transfer/protocol_options` of `aspera.conf` (auto-correction of the base RTT, for virtual machines).',
62
+ token_expired: 'Generate a new token (option `-W` of `ascp`) just before the transfer, and check that the clocks of the system that generates tokens and of the server are synchronized.',
63
+ content_protection: 'Encrypt files at rest: option `--file-crypt=encrypt` of `ascp`, with the passphrase in environment variable `ASPERA_SCP_FILEPASS`. ' \
64
+ 'The server requires it with `content_protection_required` in `transfer/encryption` of `aspera.conf`.',
65
+ insufficient_permissions: 'Check the permissions of the transfer user on the file (operating system), and the access rules of the server: `read_allowed` and `write_allowed` in `file_system/access/paths/path` of `aspera.conf`.',
66
+ out_of_disk: 'Free space on the destination storage, or check its quota, then retry the transfer with resume (option `-k` of `ascp`).',
67
+ read: storage_access,
68
+ write: storage_access,
69
+ disk_write_failed: storage_access,
70
+ device_not_connected: network_storage,
71
+ net_error: network_storage,
72
+ net_path_not_found: network_storage,
73
+ net_name_gone: network_storage,
74
+ sharing_violation: 'Another process (e.g. antivirus, indexer or application) uses the file: close it or exclude the folder from its scan, then retry.',
75
+ out_of_resources: 'Check the resources of the Windows system (memory, handles) and reduce the number of concurrent transfers.',
76
+ out_of_memory: 'Check the free memory of the system and reduce the number of concurrent transfers.',
77
+ no_buffer_mem: 'Reduce the number of concurrent transfers, or the size of UDP socket buffers (`max_sock_buffer` in `transfer/protocol_options` of `aspera.conf`).',
78
+ bind_udp_socket_failed: 'Check that the UDP port (`bind_udp_port`) and the address (`bind_ip_address`) in `transfer/protocol_options` of `aspera.conf` are valid for this system, ' \
79
+ 'and that the process is allowed to use them. The error number is in the log line.',
80
+ configuration_error: 'Correct `aspera.conf` on the system that logged the error (XML syntax and values), or use another file with option `-f` of `ascp`.',
81
+ filename_too_long: 'Shorten the destination path or the file names, so that the Windows path is not longer than 260 characters.',
82
+ remote_reset: tcp_connection,
83
+ ssh_timeout: tcp_connection,
84
+ ssh_session_died: tcp_connection,
85
+ local_reset: 'Check the firewall, antivirus and network drivers of this system: they may reset TCP connections.',
86
+ ssh_startup_failed: 'Check the host name, the TCP port of SSH (option `-P` of `ascp`, `ssh_port` of the server) and the firewalls. ' \
87
+ 'Test with option `--mode=test-login` of `ascp`. On a slow network, `ssh_startup_timeout_sec` in `aspera.conf` sets the connection timeout.',
88
+ ssh_key_authentication: 'Check the private key (option `-i` of `ascp`) or the password (environment variable `ASPERA_SCP_PASS`), and that the user is allowed to connect to the server.',
89
+ ssh_session_failed: 'Check that the SSH server accepts new sessions (load, limits), then retry. Test with option `--mode=test-login` of `ascp`.',
90
+ ssh_command_failed: 'Check that the user is configured for Aspera on the server (`aspshell`), and the name of the remote command (option `-S` of `ascp`).',
91
+ ssh_forward_failed: 'Check that the SSH server allows TCP forwarding for the user (e.g. `AllowTcpForwarding` in `sshd_config`).'
92
+ }.freeze
93
+ # Identifiers of `Session::FLAG_DIAGNOSTICS` without recommendation: nothing to do
94
+ NO_RECOMMENDATION = %w[user_abort table_died].freeze
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,344 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # cspell:words commify Isilon nohear onefs overdriven
7
+ require_relative '../session_errors'
8
+ require 'time'
9
+
10
+ module Aspera
11
+ module Ascp
12
+ class LogAnalyzer
13
+ # Diagnostics evaluated on a transfer session
14
+ class Diagnostics
15
+ UDP_TIMEOUT = 'check firewalls for UDP flood protection or UDP session timeouts'
16
+ # Control messages in `Session Statistics`: direction => description
17
+ CONTROL_DIRECTIONS = {'sndr' => 'from sender to receiver', 'rcvr' => 'from receiver to sender', 'rex' => 'requesting retransmission'}.freeze
18
+
19
+ # Flag => severity and message of diagnostic (identifier: name of flag)
20
+ FLAG_DIAGNOSTICS = {
21
+ device_not_connected: [:error, 'The device (mapped drive) is not connected.'],
22
+ timeout_establishing_udp_connection: [:error, 'Timeout establishing the UDP connection: check the UDP port and firewalls.'],
23
+ token_expired: [:error, 'The token has expired.'],
24
+ net_error: [:error, 'An unexpected network error occurred when writing to network storage.'],
25
+ net_path_not_found: [:error, 'The network path was not found.'],
26
+ net_name_gone: [:error, 'The network name is no longer available.'],
27
+ sharing_violation: [:error, 'Windows sharing violation: a file is used by another process.'],
28
+ insufficient_permissions: [:error, 'Insufficient permissions to read or write a file.'],
29
+ out_of_disk: [:error, 'The storage is out of space.'],
30
+ disk_write_failed: [:error, 'Unable to write to the storage.'],
31
+ out_of_resources: [:error, 'The Windows system is out of resources.'],
32
+ ssh_timeout: [:warning, 'The SSH session timed out, which may abort the transfer.'],
33
+ remote_reset: [:warning, 'A TCP connection was reset by the remote host, which may have ended the session.'],
34
+ local_reset: [:warning, 'A TCP connection was reset by the local host, which may have ended the session.'],
35
+ read: [:error, 'Unable to read from the storage.'],
36
+ write: [:error, 'Unable to write to the storage.'],
37
+ server_nohear_client: [:error, 'Session setup failed: the server received no UDP from the client: check the UDP port and firewalls.'],
38
+ bind_udp_socket_failed: [:error, 'The server could not bind its UDP port to start the session.'],
39
+ no_buffer_mem: [:error, 'The Windows system ran out of buffer memory: check the event log and memory.'],
40
+ client_nohear_server: [:error, 'Session setup failed: the client could not reach the server over UDP: check the UDP port and firewalls.'],
41
+ receiver_data_transfer_timeout: [:error, "The receiver received no UDP packets for 60 seconds, which ended the session: #{UDP_TIMEOUT}."],
42
+ inactivity_timer_fired: [:error, "The sender received no UDP feedback from the receiver for 60 seconds, which ended the session: #{UDP_TIMEOUT}."],
43
+ content_protection: [:error, 'The server requires content protection (encryption at rest), but the client did not provide it.'],
44
+ user_abort: [:info, 'The user cancelled the session.'],
45
+ http_fallback: [:info, 'The session fell back to HTTP.'],
46
+ table_died: [:warning, 'ascp exited at the end of the session, without logging the statistics table.'],
47
+ ssh_startup_failed: [:error, 'The SSH connection to the server could not be established.'],
48
+ ssh_key_authentication: [:error, 'SSH public key authentication failed.'],
49
+ ssh_session_failed: [:error, 'The SSH session to the server could not be opened.'],
50
+ ssh_command_failed: [:error, 'The remote command could not be started on the server through SSH.'],
51
+ ssh_forward_failed: [:error, 'The server refused TCP/IP forwarding through SSH.']
52
+ }.freeze
53
+
54
+ # @return [Array<Hash>] diagnostics collected so far
55
+ attr_reader :list
56
+
57
+ # @param session [Session] transfer session to diagnose
58
+ # @param windows_log [Boolean] log file is on Windows
59
+ def initialize(session, windows_log: false)
60
+ @session = session
61
+ @windows_log = windows_log
62
+ @thresholds = session.thresholds
63
+ @list = []
64
+ end
65
+
66
+ # Evaluate all diagnostics for the session
67
+ # @return [Array<Hash>] list of diagnostics: {id, severity, message}
68
+ def evaluate(loss: nil, slowdown: nil, sample_dir: nil, avg: nil, min_rate_high: nil, rtt: nil, srtt: nil)
69
+ d = @session.data
70
+ sample_dir ||= d[:dir]
71
+ loss ||= @session.num(:loss_rate)
72
+ diag(:old_version, :warning, "ascp version #{d[:ver]} is older than #{MIN_VERSION}: analysis may be inaccurate.") if Session.version_older?(d[:ver], MIN_VERSION)
73
+ diagnose_context
74
+ diagnose_path_mtu
75
+ diagnose_params
76
+ diag(:min_rate_high, :warning, "The minimum rate is too high: #{c(min_rate_high)} bps.") if min_rate_high
77
+ case @session.overdrive
78
+ when :storage then diag(:storage_overdrive, :warning, 'Probable cause of loss: overdriving of the storage of the receiver: check the receiver logs.')
79
+ when :network then diag(:network_overdrive, :warning, 'Probable cause of loss: shallow queues in the network, or UDP rate-limiting: check the receiver logs.')
80
+ end
81
+ diag(:brtt_reset, :info, "The base RTT was reset by ascp #{d[:brtt_reset]} times.") if d[:brtt_reset]
82
+ FLAG_DIAGNOSTICS.each { |flag, (severity, message)| diag(flag, severity, message) if @session.flag?(flag) }
83
+ diagnose_errors(loss, slowdown)
84
+ diagnose_performance(loss, sample_dir, rtt, srtt)
85
+ diagnose_loss(loss, sample_dir, avg, min_rate_high)
86
+ diagnose_stuck_nack(loss)
87
+ diagnose_quality
88
+ diagnose_rate_decline
89
+ diagnose_file_pipeline
90
+ diagnose_sender if sample_dir == 'send'
91
+ @list
92
+ end
93
+
94
+ # Error category, retries, and parameters set by the server
95
+ def diagnose_context
96
+ d = @session.data
97
+ hint = SessionErrors.hint(@session.error_category)
98
+ diag(:"error_#{@session.error_category}", :error, hint) if hint && d[:status] == 'failed'
99
+ diag(:retry, :info, "Retry of failed session #{d[:retry_of]}.") if d[:retry_of]
100
+ diag(:retried, :info, "Retried by session #{d[:retried_by]}.") if d[:retried_by]
101
+ diag(:cipher_set_by_server, :info, "Encryption cipher set by server configuration: #{d[:cipher_override].first} (requested: #{d[:cipher_override].last}).") if d[:cipher_override]
102
+ case d[:resume_downgrade]
103
+ when nil
104
+ when 'RESUME_POLICY_ATTRS' then diag(:resume_lowered, :info, 'Resume policy lowered to file attributes: checksum-based resume is not supported by the storage.')
105
+ else diag(:resume_lowered, :info, "Resume policy lowered to \"#{Session::RESUME.fetch(d[:resume_downgrade].to_i, d[:resume_downgrade])}\" as requested by the peer.")
106
+ end
107
+ diag(:job_size_changed, :info, "Job size changed from #{d[:job_size].first} to #{d[:job_size].last} to match server configuration.") if d[:job_size]
108
+ d[:vlinks]&.each { |id, vlink| diag(:vlink, :info, "Virtual link #{id} (#{vlink['type']}) limits bandwidth to #{c(vlink['capacity'])} bps.") }
109
+ diagnose_license
110
+ end
111
+
112
+ # Maximum rate of the license below the target rate, and expiry date of the license close to the session
113
+ def diagnose_license
114
+ d = @session.data
115
+ diag(:license_rate_limit, :warning, "The license limits the rate to #{c(d[:license_rate])} bps, below the target rate of #{c(@session.num(:target_rate))} bps.") if d[:license_rate] && @session.num(:target_rate) > d[:license_rate]
116
+ return unless d[:license_expires] && (started = d[:start] || d[:stop])
117
+ days = ((Time.parse(d[:license_expires]).to_i - started) / 86_400.0).floor
118
+ return if days > @thresholds[:license_expiry_days]
119
+ diag(:license_expiry, :warning, days.negative? ? "The license expired on #{d[:license_expires]}." : "The license expires on #{d[:license_expires]}, #{days} days after the start of this session.")
120
+ end
121
+
122
+ # Path MTU measured by ascp well below Ethernet (ascp probes 1492, 1006, 508 and 296 bytes)
123
+ def diagnose_path_mtu
124
+ path_mtu = @session.data[:path_mtu]
125
+ return unless path_mtu&.positive? && path_mtu < @thresholds[:small_mtu_max]
126
+ diag(
127
+ :small_mtu, :info,
128
+ "The path MTU measured by ascp is #{path_mtu} bytes, well below the 1500 bytes of Ethernet (datagram size: #{@session.data[:mtu]} bytes): a VPN, a tunnel or a device that blocks large packets may be on the path."
129
+ )
130
+ end
131
+
132
+ # Efficiency and loss of control messages, from `Session Statistics`
133
+ def diagnose_quality
134
+ d = @session.data
135
+ if d[:efficiency] && @session.num(:tx_bytes).positive? && d[:efficiency] < @thresholds[:efficiency]
136
+ diag(:low_efficiency, :warning, "Only #{d[:efficiency]}% of the data blocks sent were useful: retransmissions reduce the effective rate." \
137
+ "#{' In fixed policy, lower the target rate, or use fair policy.' if d[:policy] == 'fixed'}")
138
+ end
139
+ d[:control_loss]&.each do |direction, (sent, lost)|
140
+ next if sent < @thresholds[:control_loss_sent] || lost < @thresholds[:control_loss_percent]
141
+ diag(:control_message_loss, :warning, "#{lost}% of control messages #{CONTROL_DIRECTIONS[direction]} were lost.")
142
+ end
143
+ return unless @session.num(:rex_overflow).positive?
144
+ diag(
145
+ :rex_timer_overflow, :warning,
146
+ "The retransmission timer overflowed at least #{@session.num(:rex_overflow)} times (logged every 5 seconds at most): the retransmission RTT was very large, which delays the retransmission of lost blocks."
147
+ )
148
+ end
149
+
150
+ # Rate of the second half of the samples is lower than the first half.
151
+ # Not evaluated if the session failed, if the rate parameters changed, or if another session started and may share the link.
152
+ # The last samples are dropped: end of session, file queue drained.
153
+ def diagnose_rate_decline
154
+ return if @session.data[:status] == 'failed' || @session.params || @session.data[:shared_link]
155
+ rates = @session.rate_samples.sort_by(&:first).map(&:last)
156
+ rates = rates.first((rates.size * (1 - @thresholds[:rate_decline_end_fraction])).floor)
157
+ return if rates.size < @thresholds[:rate_decline_min_samples]
158
+ half = rates.size / 2
159
+ first = Session.average(rates.first(half))
160
+ second = Session.average(rates.last(rates.size - half))
161
+ return unless first.positive? && second < @thresholds[:rate_decline_ratio] * first
162
+ diag(:rate_decline, :warning, "The rate declined during the transfer: #{c(second)} bps on average in the second half, #{c(first)} bps in the first half (#{(100.0 * second / first).round}%).")
163
+ end
164
+
165
+ # Sender (ascp 4): limit set by the receiver, and storage read rate
166
+ def diagnose_sender
167
+ d = @session.data
168
+ if @session.receiver_causes.any?
169
+ total = @session.receiver_causes.values.sum
170
+ causes = @session.receiver_causes.sort_by { |_, count| -count }.map { |cause, count| "#{cause} (#{(100.0 * count / total).round}% of samples)" }
171
+ diag(:receiver_limit, :info, "Rate limited by the receiver: #{causes.join(', ')}.")
172
+ end
173
+ calculated = Session.average(@session.series[:calc]).to_i
174
+ return unless d[:disk_read_rate] && calculated.positive? && d[:disk_read_rate] < calculated
175
+ diag(
176
+ :slow_storage_read, :warning,
177
+ "Reading from storage (#{c(d[:disk_read_rate])} bps) is slower than the calculated rate (#{c(calculated)} bps): sender storage limits the transfer."
178
+ )
179
+ end
180
+
181
+ def diagnose_params
182
+ return unless @session.params
183
+ p = @session.params
184
+ changes = []
185
+ changes.push("Policy changed from #{p[:policy_old]} to #{p[:policy]}.") if p.key?(:policy)
186
+ changes.push("Target rate changed from #{c(p[:target_rate_old])} bps to #{c(p[:target_rate])} bps.") if p.key?(:target_rate)
187
+ changes.push("Minimum rate changed from #{c(p[:min_rate_old])} bps to #{c(p[:min_rate])} bps.") if p.key?(:min_rate)
188
+ diag(:rate_change, :info, "#{p[:multiple] ? 'Last of multiple parameter changes' : 'Parameter change'}#{' by peer' if p[:peer]}: #{changes.join(' ')}")
189
+ end
190
+
191
+ def diagnose_errors(loss, slowdown)
192
+ d = @session.data
193
+ if @session.flag?(:out_of_memory)
194
+ diag(:out_of_memory, :error, "The system is running out of memory#{' (on Isilon, possibly because of asperacentral)' if d[:os].to_s.match?(/onefs/i)}.")
195
+ end
196
+ if @session.flag?(:as_buf_io)
197
+ cause = "probably because of the high packet loss#{', due to fixed policy' if d[:policy] == 'fixed'}" if loss > 0.9
198
+ diag(:ssh_session_died, :warning, "The SSH session died, which may abort the transfer#{": #{cause}" if cause}.")
199
+ end
200
+ if @session.num(:no_data).positive?
201
+ causes = "#{'disk or network constraints on the rate, ' if slowdown}rate-limiting, a firewall block, or a network outage"
202
+ diag(:reception_interruption, :warning, "Data reception was interrupted for up to #{d[:no_data]} seconds (#{d[:no_data_total]} seconds in total): possibly caused by #{causes}.")
203
+ end
204
+ if @session.num(:no_feedback).positive?
205
+ diag(
206
+ :feedback_interruption, :warning,
207
+ "The sender received no packet from the receiver for up to #{d[:no_feedback]} seconds (#{d[:no_feedback_total]} seconds in total): possibly caused by rate-limiting, a firewall block, or a network outage."
208
+ )
209
+ end
210
+ diag(:filename_too_long, :error, "#{d[:filename_too_long]} files failed because their Windows path is longer than 260 characters.") if d[:filename_too_long] && (@windows_log || @session.windows?)
211
+ diag(:configuration_error, :error, "Error in aspera.conf: #{d[:conf_parse]}") if d[:conf_parse]
212
+ # Session ended without `Session Stop`: show the error logged just before, if any
213
+ diag(:last_error, :error, "Last error before ascp stopped logging: #{d[:last_error]}") if d[:exit] && d[:stop].nil? && d[:last_error] && d[:last_error_time].to_i >= @session.num(:last) - @thresholds[:last_error_window]
214
+ if d[:ssh_disconnect]
215
+ # libssh2 error -9: timeout
216
+ diag(
217
+ :ssh_not_closed, :warning,
218
+ "The SSH connection to the server was not closed properly: #{d[:ssh_disconnect]}#{' (timeout: no answer from server)' if d[:ssh_disconnect].end_with?('rc=-9')}"
219
+ )
220
+ end
221
+ if @session.num(:gap).positive?
222
+ diag(:io_wait, :warning, "ascp was blocked (IO wait) for #{d[:gap]} seconds, until #{Time.at(d[:gap_last]).getlocal(@session.timezone).ctime}#{', with an SSH timeout' if @session.flag?(:ssh_timeout)}.")
223
+ end
224
+ diag(:udp_flood_protection, :warning, 'No route to host during the transfer: possibly caused by UDP flood protection of a firewall: check the logs.') if @session.flag?(:send_no_route) && @session.num(:tx_bytes).nonzero? && !@session.flag?(:as_buf_io)
225
+ diag(:local_port_changed, :info, "Local port changed from #{d[:start_l_port]} to #{d[:l_port]} during the session.") if d[:start_l_port]
226
+ end
227
+
228
+ def diagnose_performance(loss, sample_dir, rtt, srtt)
229
+ d = @session.data
230
+ fair_only = (' Storage rate control requires fair policy.' if d[:policy] == 'fixed')
231
+ diag(:disk_write_speed_exceeded, :warning, "The disk write speed was exceeded.#{fair_only}") if @session.flag?(:exceeded)
232
+ if @session.flag?(:vm) && !d[:brtt_reset]
233
+ rate = " The calculated rate dropped to #{c(d[:c_rate])} bps, with a base RTT of #{d[:brtt]} ms." if sample_dir == 'receive'
234
+ diag(:clock_problem, :warning, "This system (ascp #{d[:ver]}) may run on a VM, or have another clock problem, which affected the transfer.#{rate}")
235
+ elsif @session.flag?(:vm_peer)
236
+ others = [
237
+ (' Reverse path congestion inference could also cause it: check the peer logs.' if @session.flag?(:disable_reverse)),
238
+ (' Slow storage could also lower the rate.' if d[:disk_max]),
239
+ (" The calculated rate dropped to #{c(d[:vm_c_rate])} bps." if sample_dir == 'send')
240
+ ].join
241
+ diag(:peer_clock_problem, :warning, "The peer may run on a VM, or have another clock problem, which affected the transfer.#{others}")
242
+ end
243
+ diag(:reverse_path_inference_disabled, :info, 'Reverse path congestion inference is disabled for this transfer.') if @session.flag?(:no_reverse_infer)
244
+ if @session.flag?(:disable_reverse) && !@session.flag?(:no_reverse_infer) && !d[:brtt_reset]
245
+ rate = " The calculated rate dropped to #{c(d[:n_rate])} bps, with a base RTT of #{d[:brtt]} ms." if sample_dir == 'receive'
246
+ diag(:reverse_path_skew, :warning, "Reverse path congestion skew lowered the rate: check the logs, and disable reverse path congestion inference if appropriate.#{rate}")
247
+ end
248
+ diag(:slow_disk_write, :warning, "The disk write speed of the receiver was too slow #{d[:disk_max]} times.") if d[:disk_max]
249
+ diag(:disk_buffers, :warning, "The disk write buffer depth reached #{d[:buffers]}.#{fair_only}") if d[:buffers]
250
+ if @session.flag?(:rex_buffer_full)
251
+ diag(
252
+ :retransmission_buffer_full, :warning,
253
+ "The retransmission buffer was full: possibly severe packet loss caused by rate-limiting or policing#{', or by fixed policy' unless d[:policy] == 'fair'}."
254
+ )
255
+ end
256
+ if @session.num(:big_rtt).positive?
257
+ cause =
258
+ if @session.flag?(:rex_buffer_full) then 'probably because of a full retransmission buffer'
259
+ elsif @session.flag?(:exceeded) then 'probably because of slow disk write'
260
+ elsif loss > 0.3 then 'probably because of the high packet loss'
261
+ else 'cause not apparent: investigate'
262
+ end
263
+ diag(:high_rtt, :warning, "The round trip time is excessive (#{d[:big_rtt]} ms): #{cause}.")
264
+ end
265
+ if d[:big_rex_srtt]
266
+ rex = @session.series[:rex_srtt]
267
+ median = Session.median(rex)
268
+ diag(
269
+ :high_retransmission_rtt, :warning,
270
+ "The retransmission round trip time is excessive: maximum #{d[:big_rex_srtt]} ms, average #{Session.average(rex).to_i} ms, median #{median == median.to_i ? median.to_i : median} ms."
271
+ )
272
+ end
273
+ return unless d[:atm] && rtt&.first.to_i.nonzero? && loss * 100 > 20
274
+ path = ' The path may contain a rate-limiting firewall or router, or an ATM switch, causing excessive packet loss.' if srtt&.first.to_i.nonzero? && @session.num(:brtt).fdiv(srtt.first) >= 0.95
275
+ diag(:network_overdriven, :warning, "The network capacity is severely overdriven: lower the target rate#{' or use fair policy' if d[:policy] == 'fixed'}.#{path}")
276
+ end
277
+
278
+ def diagnose_loss(loss, sample_dir, avg, min_rate_high)
279
+ d = @session.data
280
+ loss_pct = loss * 100
281
+ if loss_pct >= 10 && (d[:disk_max] || d[:buffers] == 12)
282
+ if min_rate_high
283
+ diag(:min_rate_overdrive, :warning, 'Loss and disk overdrive possibly caused by a minimum rate too high.')
284
+ else
285
+ diag(:slow_storage_loss, :warning, "Loss possibly caused by slow storage write on the receiver#{" (maximum disk constraint reached #{d[:disk_max]} times)" if d[:disk_max]}.")
286
+ end
287
+ end
288
+ return unless sample_dir == 'send'
289
+ receiver_min = @session.constraint.dig('Receiver', :min_rate)
290
+ if receiver_min && @session.num(:target_rate).positive? && receiver_min.fdiv(d[:target_rate]) > 0.9 && loss_pct > 30
291
+ diag(
292
+ :network_overdriven, :warning,
293
+ 'The network capacity is severely overdriven: the path may contain a rate-limiting firewall or router, or an ATM switch, causing excessive packet loss: check the logs of the receiver.'
294
+ )
295
+ end
296
+ return unless min_rate_high && min_rate_high > avg && loss_pct > 20
297
+ diag(:min_rate_overdrive, :warning, 'The network capacity is severely overdriven, possibly because of a minimum rate too high: in fair policy, set it low (e.g. 20 Kbps).')
298
+ end
299
+
300
+ def diagnose_stuck_nack(loss)
301
+ return if @session.data[:status] == 'success'
302
+ stuck = 'Retransmission requests (NACKs) are stuck'
303
+ message =
304
+ if @session.num(:stuck_nack_count) >= 5
305
+ if loss >= 0.4 then "#{stuck}, probably because of the packet loss."
306
+ elsif @session.num(:stuck_rex_count).zero? then "#{stuck}: #{@session.data[:stuck_nack_count]} samples with #{@session.data[:stuck_nack]} pending requests."
307
+ elsif @session.data[:stuck_rex_count] > 5 then "#{stuck}."
308
+ end
309
+ elsif @session.num(:sender_stuck_nack_count) >= 5 && loss.nonzero? then "#{stuck} on the sender, and data packets were lost."
310
+ elsif @session.flag?(:send_nack_stuck) && loss < 0.9 then "#{stuck} on the sender#{', probably lost because of the packet loss' if loss > 0.4}."
311
+ end
312
+ diag(:stuck_nack, :warning, message) if message
313
+ end
314
+
315
+ # Small files and file-queue diagnostics, from `DS Qs` lines
316
+ def diagnose_file_pipeline
317
+ count = @session.data[:transfer_stop_count].to_i
318
+ return if count < @thresholds[:small_files_min_count]
319
+ mean = Session.average(@session.sizes).to_i
320
+ return if mean.zero? || mean >= @thresholds[:small_files_max_mean]
321
+ diag(
322
+ :small_files, :warning,
323
+ "#{c(count)} files with a mean size of #{c(mean)} bytes: " \
324
+ 'the per-file overhead (open, attributes, close) dominates and limits throughput.'
325
+ )
326
+ end
327
+
328
+ private
329
+
330
+ # Add a diagnostic
331
+ # @param id [Symbol] identifier
332
+ # @param severity [Symbol] `error`, `warning` or `info`
333
+ # @param message [String] description
334
+ def diag(id, severity, message)
335
+ @list.push({'id' => id.to_s, 'severity' => severity.to_s, 'message' => message})
336
+ end
337
+
338
+ def c(value)
339
+ Session.commify(value)
340
+ end
341
+ end
342
+ end
343
+ end
344
+ end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ # SPDX-FileCopyrightText: IBM Corp.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # cspell:words getlocal MSGID PROCID rsyslog
7
+ module Aspera
8
+ module Ascp
9
+ class LogAnalyzer
10
+ # Path that designates the standard input
11
+ STDIN_PATH = '-'
12
+ # Formats and timestamps of log lines: shared by LogFile and Records, which define `@path` and `@timezone` (nil: time zone of the analyst)
13
+ module Lines
14
+ MONTHS = %w[Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec].freeze
15
+ # Line of ascp: FASP message, or syslog line of process `ascp`
16
+ ASCP_LINE = /\bFASP\b|\sascp\d*\[\d+\]:/
17
+ # Gap in seconds between lines beyond which a syslog line may be in another year
18
+ HALF_YEAR = 180 * 86_400
19
+
20
+ # @return [Boolean] true if a line of the file is an ascp log line
21
+ def ascp?
22
+ @ascp || false
23
+ end
24
+
25
+ private
26
+
27
+ # Remember that the file contains ascp log lines
28
+ def track_ascp(line)
29
+ @ascp ||= line.match?(ASCP_LINE)
30
+ end
31
+
32
+ # Fix variants of log formats
33
+ def normalize(line)
34
+ if line.match?(/\A\d+-\d+-\d+T/)
35
+ line = line.sub(/(\d+-\d+-\d+)T(\d+:\d+:\d+)(?:\.\d+)?[+-]\d+:\d+(.*)$/, '\1 \2 \3') # ISO 8601 with time zone (rsyslog)
36
+ line = line.sub(/^(\d+-\d+-\d+)T(\d+:\d+:\d+)Z(.*)$/, '\1 \2 \3') # ISO 8601 UTC
37
+ end
38
+ line = line.sub(/(^.*?\s+ascp)\s+(\d+)\s+-\s+-(.*)/, '\1[\2]\3') if line.include?(' - -') # RFC 5424: APP-NAME PROCID MSGID
39
+ line = line.sub(/^<\d+>\d\s+(.*)/, '\1') if line.start_with?('<') # RFC 5424: PRI VERSION
40
+ line
41
+ end
42
+
43
+ # @param time [Integer, nil] Epoch time
44
+ # @return [String, nil] ISO 8601 time, in the time zone of the log lines
45
+ def iso(time)
46
+ Time.at(time).getlocal(@timezone).iso8601 if time
47
+ end
48
+
49
+ # @return [Integer, nil] epoch time of `Mon DD HH:MM:SS` (year deduced) or `YYYY-MM-DD HH:MM:SS`, in the time zone of the log lines
50
+ def parse_time(text)
51
+ return if text.nil?
52
+ @time_cache ||= {}
53
+ @time_cache.clear if @time_cache.size > 10_000
54
+ @time_cache[text] ||=
55
+ if (m = text.match(/^(\w+)\s+(\d+)\s+(\d+):(\d+):(\d+)/)) && (month = MONTHS.index(m[1]))
56
+ month += 1
57
+ day = m[2].to_i
58
+ Time.new(syslog_year(month, day), month, day, m[3].to_i, m[4].to_i, m[5].to_i, @timezone).to_i
59
+ elsif (m = text.match(/^(\d+)-(\d+)-(\d+)\s+(\d+):(\d+):(\d+)/))
60
+ Time.new(*m.captures.map(&:to_i), @timezone).to_i
61
+ end
62
+ end
63
+
64
+ # Syslog lines have no year. The log ends before the modification date of the file (the current date for the standard input):
65
+ # the year of a line is the latest one that is not after this date. So the year decreases when going back past January
66
+ # (a log from December to January analyzed in March: December is of the previous year).
67
+ # @return [Integer] year of the line
68
+ def syslog_year(month, day)
69
+ @reference ||= (@path == STDIN_PATH ? Time.now : File.mtime(@path)).getlocal(@timezone)
70
+ month > @reference.month || (month == @reference.month && day > @reference.day) ? @reference.year - 1 : @reference.year
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end