aspera-cli 4.27.5 → 4.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +160 -4
  4. data/CONTRIBUTING.md +4 -2
  5. data/README.md +1 -1
  6. data/bin/asession +1 -1
  7. data/docs/README.md +1086 -331
  8. data/docs/test-mcp-with-ai.md +5 -3
  9. data/lib/aspera/agent/direct.rb +15 -4
  10. data/lib/aspera/agent/factory.rb +2 -1
  11. data/lib/aspera/agent/node.rb +1 -1
  12. data/lib/aspera/api/cos_node.rb +1 -1
  13. data/lib/aspera/api/faspex.rb +1 -1
  14. data/lib/aspera/api/node.rb +6 -2
  15. data/lib/aspera/ascp/installation.rb +47 -11
  16. data/lib/aspera/ascp/management.rb +89 -43
  17. data/lib/aspera/cli/ascp_actions.rb +5 -19
  18. data/lib/aspera/cli/bootstrapper.rb +1 -2
  19. data/lib/aspera/cli/command_registry.rb +43 -3
  20. data/lib/aspera/cli/command_spec.rb +3 -0
  21. data/lib/aspera/cli/completion/ascli.bash +39 -5
  22. data/lib/aspera/cli/completion/ascli.fish +16 -8
  23. data/lib/aspera/cli/completion/ascli.ps1 +41 -0
  24. data/lib/aspera/cli/completion/ascli.zsh +20 -4
  25. data/lib/aspera/cli/completion_actions.rb +360 -0
  26. data/lib/aspera/cli/error.rb +42 -0
  27. data/lib/aspera/cli/extended_value.rb +30 -5
  28. data/lib/aspera/cli/formatter.rb +118 -56
  29. data/lib/aspera/cli/hints.rb +0 -1
  30. data/lib/aspera/cli/http.rb +6 -5
  31. data/lib/aspera/cli/info.rb +10 -4
  32. data/lib/aspera/cli/mailer.rb +1 -1
  33. data/lib/aspera/cli/mcp/http_servlet.rb +156 -0
  34. data/lib/aspera/cli/mcp/secrets.rb +143 -0
  35. data/lib/aspera/cli/mcp/tool.rb +231 -0
  36. data/lib/aspera/cli/option_value.rb +9 -1
  37. data/lib/aspera/cli/options.schema.yaml +390 -39
  38. data/lib/aspera/cli/parser.rb +14 -5
  39. data/lib/aspera/cli/plugins/aoc.rb +8 -11
  40. data/lib/aspera/cli/plugins/base.rb +36 -15
  41. data/lib/aspera/cli/plugins/config.rb +15 -44
  42. data/lib/aspera/cli/plugins/console.rb +3 -1
  43. data/lib/aspera/cli/plugins/factory.rb +1 -1
  44. data/lib/aspera/cli/plugins/faspex5.rb +9 -19
  45. data/lib/aspera/cli/plugins/faspio.rb +36 -5
  46. data/lib/aspera/cli/plugins/httpgw.rb +1 -0
  47. data/lib/aspera/cli/plugins/mcp.rb +177 -118
  48. data/lib/aspera/cli/plugins/node.rb +14 -8
  49. data/lib/aspera/cli/plugins/orchestrator.rb +351 -38
  50. data/lib/aspera/cli/plugins/server.rb +3 -1
  51. data/lib/aspera/cli/plugins/shares.rb +205 -35
  52. data/lib/aspera/cli/preset_actions.rb +15 -7
  53. data/lib/aspera/cli/preset_manager.rb +24 -2
  54. data/lib/aspera/cli/prompt.rb +3 -1
  55. data/lib/aspera/cli/result.rb +17 -4
  56. data/lib/aspera/cli/runner.rb +133 -43
  57. data/lib/aspera/cli/security.rb +134 -0
  58. data/lib/aspera/cli/table_fitter.rb +116 -0
  59. data/lib/aspera/cli/terminal_formatter.rb +23 -34
  60. data/lib/aspera/cli/transfer_agent.rb +4 -1
  61. data/lib/aspera/cli/vault_manager.rb +6 -5
  62. data/lib/aspera/cli/version.rb +1 -1
  63. data/lib/aspera/cli/web_secret.html.erb +61 -0
  64. data/lib/aspera/cli/web_secret.rb +220 -0
  65. data/lib/aspera/cli/wizard.rb +19 -9
  66. data/lib/aspera/command_line_builder.rb +26 -0
  67. data/lib/aspera/data_repository.rb +38 -24
  68. data/lib/aspera/duration.rb +42 -0
  69. data/lib/aspera/environment.rb +88 -24
  70. data/lib/aspera/exec_spec.rb +6 -5
  71. data/lib/aspera/faspex_gw.rb +98 -32
  72. data/lib/aspera/faspex_postproc.rb +187 -44
  73. data/lib/aspera/formatter_interface.rb +0 -1
  74. data/lib/aspera/id_generator.rb +2 -1
  75. data/lib/aspera/json_rpc/client.rb +51 -32
  76. data/lib/aspera/json_rpc/version.rb +1 -0
  77. data/lib/aspera/keychain/base.rb +5 -0
  78. data/lib/aspera/keychain/encrypted_hash.rb +6 -2
  79. data/lib/aspera/keychain/factory.rb +2 -3
  80. data/lib/aspera/keychain/hashicorp_vault.rb +7 -4
  81. data/lib/aspera/keychain/macos_security.rb +119 -13
  82. data/lib/aspera/keychain/one_password_api.rb +8 -3
  83. data/lib/aspera/keychain/one_password_base.rb +14 -9
  84. data/lib/aspera/keychain/one_password_cli.rb +4 -2
  85. data/lib/aspera/log.rb +3 -6
  86. data/lib/aspera/markdown.rb +152 -115
  87. data/lib/aspera/mime.rb +6 -0
  88. data/lib/aspera/nagios.rb +3 -1
  89. data/lib/aspera/node_emulator.rb +98 -101
  90. data/lib/aspera/oauth/{base.rb → client/base.rb} +5 -4
  91. data/lib/aspera/oauth/{boot.rb → client/boot.rb} +3 -2
  92. data/lib/aspera/oauth/{factory.rb → client/factory.rb} +7 -0
  93. data/lib/aspera/oauth/{generic.rb → client/generic.rb} +1 -1
  94. data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +1 -1
  95. data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +1 -1
  96. data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +1 -1
  97. data/lib/aspera/oauth/{web.rb → client/web.rb} +2 -2
  98. data/lib/aspera/oauth/server/authorize.html.erb +34 -0
  99. data/lib/aspera/oauth/server/pkce.rb +426 -0
  100. data/lib/aspera/orchestrator_workflow_checker.rb +561 -0
  101. data/lib/aspera/preview/terminal.rb +3 -2
  102. data/lib/aspera/rainbow.rb +27 -1
  103. data/lib/aspera/rest/aspera_errors.rb +51 -59
  104. data/lib/aspera/rest/client.rb +21 -14
  105. data/lib/aspera/rest/error_analyzer.rb +9 -0
  106. data/lib/aspera/rest/list.rb +6 -6
  107. data/lib/aspera/rest/util.rb +7 -1
  108. data/lib/aspera/rest.rb +0 -1
  109. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -2
  110. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4001 -0
  111. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
  112. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +35 -35
  113. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
  114. data/lib/aspera/schema/documentation.rb +50 -78
  115. data/lib/aspera/schema/reader.rb +39 -11
  116. data/lib/aspera/schema/registry.rb +8 -2
  117. data/lib/aspera/secret_hider.rb +94 -15
  118. data/lib/aspera/secret_strength.rb +114 -0
  119. data/lib/aspera/servlet_helpers.rb +111 -0
  120. data/lib/aspera/sync/args.schema.yaml +3 -1
  121. data/lib/aspera/sync/conf.schema.yaml +16 -6
  122. data/lib/aspera/sync/operations.rb +21 -1
  123. data/lib/aspera/temp_file_manager.rb +79 -20
  124. data/lib/aspera/transfer/parameters.rb +5 -0
  125. data/lib/aspera/transfer/spec.rb +52 -12
  126. data/lib/aspera/transfer/spec.schema.yaml +8 -5
  127. data/lib/aspera/web_auth/index.html.erb +16 -0
  128. data/lib/aspera/web_auth/server.rb +18 -10
  129. data/lib/aspera/web_page.html.erb +262 -0
  130. data/lib/aspera/web_page.rb +61 -0
  131. data/lib/aspera/web_server_simple.rb +76 -10
  132. data.tar.gz.sig +0 -0
  133. metadata +47 -32
  134. metadata.gz.sig +0 -0
  135. data/lib/aspera/cli/mcp_tool.rb +0 -189
  136. data/lib/aspera/oauth.rb +0 -9
  137. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
  138. data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
  139. data/lib/aspera/web_auth/index.html +0 -147
@@ -1,141 +1,178 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/assert'
4
+
3
5
  module Aspera
4
6
  # Formatting for Markdown
5
- class Markdown
7
+ module Markdown
8
+ extend self
9
+
6
10
  # Matches: **bold**, `code`, or an HTML entity (&, ©, 💩)
7
- FORMATS = /(?:\*\*(?<bold>[^*]+?)\*\*)|(?:`(?<code>[^`]+)`)|&(?<entity>(?:[A-Za-z][A-Za-z0-9]{1,31}|#\d{1,7}|#x[0-9A-Fa-f]{1,6}));/m
11
+ INLINE_MARKUP_RE = /(?:\*\*(?<bold>[^*]+?)\*\*)|(?:`(?<code>[^`]+)`)|&(?<entity>(?:[A-Za-z][A-Za-z0-9]{1,31}|#\d{1,7}|#x[0-9A-Fa-f]{1,6}));/m
12
+ # HTML line-break tag used in Markdown table cells to separate multi-line values.
8
13
  HTML_BREAK = '<br/>'
14
+ # Matches a Markdown heading line (e.g. `## Title`); group 1 = `#` chars, group 2 = text.
15
+ HEADING_RE = /^(\#{1,6})[ \t]+(\S.*)$/
16
+ # Matches the start or end of a fenced code block; group 1 = fence (``` or ~~~).
17
+ FENCE_RE = /^ {0,3}(`{3,}|~{3,})/
18
+ # Maximum column width (in characters) for generated Markdown tables.
19
+ COL_WIDTH = 80
20
+ # Accepted admonition types.
21
+ ADMONITION_TYPES = %w[NOTE CAUTION WARNING IMPORTANT TIP INFO].freeze
9
22
 
10
- class << self
11
- COL_WIDTH = 80
12
-
13
- # Convert a Markdown heading text to a GitHub-flavoured anchor.
14
- # Rules: downcase, keep letters/digits/spaces/hyphens, replace spaces with hyphens.
15
- # Duplicate anchors are disambiguated by appending -1, -2, … (pass a seen Hash to track).
16
- # @param text [String] raw heading text (without leading # and spaces)
17
- # @param seen [Hash{String=>Integer}, nil] mutable counter; pass the same Hash across a document
18
- # @return [String] anchor slug (without leading #)
19
- def heading_to_anchor(text, seen: nil)
20
- slug = text
21
- .downcase
22
- .gsub(/[`*_]/, '') # strip inline code/bold/italic markers
23
- .gsub(/&[a-z]+;/, '') # strip HTML entities
24
- .gsub(/[^\w\s-]/, '') # keep word chars, spaces, hyphens
25
- .gsub(/\s+/, '-') # spaces → hyphens
26
- .squeeze('-') # collapse consecutive hyphens
27
- .strip
28
- if seen
29
- count = seen[slug].to_i
30
- seen[slug] = count + 1
31
- slug = "#{slug}-#{count}" if count > 0
32
- end
33
- slug
34
- end
23
+ # --- Parsing ---
35
24
 
36
- # Extract the table of contents from a Markdown document.
37
- # @param content [String] full Markdown source
38
- # @return [Array<Hash>] array of { level, title, anchor }
39
- HEADING_RE = /^(\#{1,6})\s+(.+)$/
40
-
41
- def toc(content)
42
- seen = {}
43
- content.each_line.filter_map do |line|
44
- m = line.match(HEADING_RE)
45
- next unless m
46
- title = m[2].strip
47
- {level: m[1].length, title: title, anchor: heading_to_anchor(title, seen: seen)}
48
- end
25
+ # Convert a Markdown heading text to a GitHub-flavoured anchor.
26
+ # Rules: downcase, keep letters/digits/spaces/hyphens, replace spaces with hyphens.
27
+ # Duplicate anchors are disambiguated by appending -1, -2, … (pass a seen Hash to track).
28
+ # @param text [String] raw heading text (without leading # and spaces)
29
+ # @param seen [Hash{String=>Integer}, nil] mutable counter; pass the same Hash across a document
30
+ # @return [String] anchor slug (without leading #)
31
+ def heading_to_anchor(text, seen: nil)
32
+ slug = text
33
+ .downcase
34
+ .gsub(/[`*_]/, '') # strip inline code/bold/italic markers
35
+ .gsub(/&[a-z]+;/, '') # strip HTML entities
36
+ .gsub(/[^\w\s-]/, '') # keep word chars, spaces, hyphens
37
+ .gsub(/\s+/, '-') # spaces → hyphens
38
+ .squeeze('-') # collapse consecutive hyphens
39
+ .strip
40
+ if seen
41
+ count = seen[slug].to_i
42
+ seen[slug] = count + 1
43
+ slug = "#{slug}-#{count}" if count > 0
49
44
  end
45
+ slug
46
+ end
50
47
 
51
- # Extract the content of a single section (heading + body until next heading of same/higher level).
52
- # @param content [String] full Markdown source
53
- # @param anchor [String] GitHub anchor slug (without #)
54
- # @return [String, nil] the section content, or nil if not found
55
- def extract_section(content, anchor)
56
- seen = {}
57
- section_level = nil
58
- result = []
59
- content.each_line do |line|
60
- m = line.match(HEADING_RE)
61
- if m
62
- slug = heading_to_anchor(m[2].strip, seen: seen)
63
- if section_level.nil?
64
- # not yet found: check if this heading matches
65
- next unless slug == anchor
66
- section_level = m[1].length
67
- elsif m[1].length <= section_level
68
- # already in section: stop at same/higher level heading
69
- break
70
- end
48
+ # Iterate on the lines of a Markdown document, with the heading match of each line.
49
+ # A line in a fenced code block is not a heading (e.g. a shell comment `# ...`).
50
+ # @param content [String] full Markdown source
51
+ # @yieldparam line [String] the line
52
+ # @yieldparam heading [MatchData, nil] match of HEADING_RE, or nil if not a heading
53
+ def each_line_with_heading(content)
54
+ fence = nil
55
+ content.each_line do |line|
56
+ if (f = line.match(FENCE_RE))
57
+ # a code block ends with the same kind of fence, at least as long, without info string
58
+ if fence.nil?
59
+ fence = f[1]
60
+ elsif f[1].start_with?(fence) && line.strip.eql?(f[1])
61
+ fence = nil
71
62
  end
72
- result << line if section_level
63
+ yield(line, nil)
64
+ else
65
+ yield(line, fence.nil? ? line.match(HEADING_RE) : nil)
73
66
  end
74
- result.empty? ? nil : result.join
75
67
  end
68
+ end
76
69
 
77
- # Generate markdown from the provided 2D table
78
- # @param table [Array<Array<String>>] 2D array of strings
79
- # @return [String] markdown table
80
- def table(table)
81
- # get max width of each columns
82
- col_widths = table.transpose.map do |col|
83
- [col.flat_map { |c| c.to_s.delete('`').split(HTML_BREAK).map(&:size) }.max, COL_WIDTH].min
84
- end
85
- # do not modify caller's table
86
- table = table.dup
87
- headings = table.shift
88
- table.unshift(col_widths.map { |col_width| '-' * col_width })
89
- table.unshift(headings)
90
- lines = table.map { |line| "| #{line.map { |i| i.to_s.gsub('\\', '\\\\').gsub('|', '\|') }.join(' | ')} |\n" }
91
- lines[1] = lines[1].tr(' ', '-')
92
- return lines.join.chomp
70
+ # Extract the table of contents from a Markdown document.
71
+ # @param content [String] full Markdown source
72
+ # @param max_level [Integer, nil] maximum level of headings, or nil for all
73
+ # @return [Array<Hash>] array of { level, title, anchor } (String keys, as other results: `--select` applies)
74
+ def toc(content, max_level: nil)
75
+ seen = {}
76
+ result = []
77
+ each_line_with_heading(content) do |_line, m|
78
+ next unless m
79
+ title = m[2].strip
80
+ # anchor of every heading, so that duplicate anchors are numbered as in the whole document
81
+ anchor = heading_to_anchor(title, seen: seen)
82
+ result.push({'level' => m[1].length, 'title' => title, 'anchor' => anchor}) if max_level.nil? || m[1].length <= max_level
93
83
  end
84
+ result
85
+ end
94
86
 
95
- # Generate markdown list from the provided list
96
- # @param items [Array<String>] list of items
97
- # @return [String] markdown unordered list
98
- def list(items)
99
- items.map { |i| "- #{i}" }.join("\n")
87
+ # Extract the content of a single section (heading + body until next heading of same/higher level).
88
+ # @param content [String] full Markdown source
89
+ # @param anchor [String] GitHub anchor slug (without #)
90
+ # @return [String, nil] the section content, or nil if not found
91
+ def extract_section(content, anchor)
92
+ seen = {}
93
+ section_level = nil
94
+ result = []
95
+ each_line_with_heading(content) do |line, m|
96
+ if m
97
+ slug = heading_to_anchor(m[2].strip, seen: seen)
98
+ if section_level.nil?
99
+ # not yet found: check if this heading matches
100
+ next unless slug == anchor
101
+ section_level = m[1].length
102
+ elsif m[1].length <= section_level
103
+ # already in section: stop at same/higher level heading
104
+ break
105
+ end
106
+ end
107
+ result << line if section_level
100
108
  end
109
+ result.empty? ? nil : result.join
110
+ end
101
111
 
102
- # Generate a markdown heading
103
- # @param title [String] heading text
104
- # @param level [Integer] heading level (1–6)
105
- # @return [String] markdown heading
106
- def heading(title, level: 1)
107
- "#{'#' * level} #{title}\n\n"
108
- end
112
+ # --- Generation ---
109
113
 
110
- # Generate a GitHub-flavoured admonition block
111
- # @param lines [Array<String>] lines of the admonition body
112
- # @param type [String] admonition type: NOTE, CAUTION, WARNING, IMPORTANT, TIP, INFO
113
- # @return [String] markdown admonition block
114
- def admonition(lines, type: 'INFO')
115
- "> [!#{type}]\n#{lines.map { |l| "> #{l}" }.join("\n")}\n\n"
116
- end
114
+ # Wrap text in inline code backticks
115
+ # @param text [String] text to wrap
116
+ # @return [String] inline code span
117
+ def icode(text)
118
+ "`#{text}`"
119
+ end
117
120
 
118
- # Generate a fenced code block
119
- # @param lines [Array<String>] lines of code
120
- # @param type [String] language identifier for syntax highlighting
121
- # @return [String] markdown fenced code block
122
- def code(lines, type: 'shell')
123
- "```#{type}\n#{lines.join("\n")}\n```\n\n"
124
- end
121
+ # Wrap text in a markdown paragraph (trailing blank line)
122
+ # @param text [String] paragraph content
123
+ # @return [String] paragraph with trailing newlines
124
+ def paragraph(text)
125
+ "#{text}\n\n"
126
+ end
125
127
 
126
- # Wrap text in inline code backticks
127
- # @param text [String] text to wrap
128
- # @return [String] inline code span
129
- def icode(text)
130
- "`#{text}`"
131
- end
128
+ # Generate a markdown heading
129
+ # @param title [String] heading text
130
+ # @param level [Integer] heading level (1–6)
131
+ # @return [String] markdown heading
132
+ def heading(title, level: 1)
133
+ paragraph("#{'#' * level} #{title}")
134
+ end
135
+
136
+ # Generate a fenced code block
137
+ # @param lines [Array<String>] lines of code
138
+ # @param type [String] language identifier for syntax highlighting
139
+ # @return [String] markdown fenced code block
140
+ def code(lines, type: 'shell')
141
+ paragraph("```#{type}\n#{lines.join("\n")}\n```")
142
+ end
143
+
144
+ # Generate a GitHub-flavoured admonition block
145
+ # @param lines [Array<String>] lines of the admonition body
146
+ # @param type [String] admonition type, one of {ADMONITION_TYPES}
147
+ # @return [String] markdown admonition block
148
+ def admonition(lines, type: 'INFO')
149
+ Aspera.assert_values(type, ADMONITION_TYPES) { 'admonition type' }
150
+ paragraph((["[!#{type}]"] + lines).map { |l| "> #{l}" }.join("\n"))
151
+ end
152
+
153
+ # Generate markdown list from the provided list
154
+ # @param items [Array<String>] list of items
155
+ # @return [String] markdown unordered list
156
+ def list(items)
157
+ items.map { |i| "- #{i}" }.join("\n")
158
+ end
132
159
 
133
- # Wrap text in a markdown paragraph (trailing blank line)
134
- # @param text [String] paragraph content
135
- # @return [String] paragraph with trailing newlines
136
- def paragraph(text)
137
- "#{text}\n\n"
160
+ # Generate markdown from the provided 2D table
161
+ # @param table [Array<Array<String>>] 2D array of strings
162
+ # @return [String] markdown table
163
+ def table(table)
164
+ # get max width of each columns
165
+ col_widths = table.transpose.map do |col|
166
+ [col.flat_map { |c| c.to_s.delete('`').split(HTML_BREAK).map(&:size) }.max, COL_WIDTH].min
138
167
  end
168
+ # do not modify caller's table
169
+ table = table.dup
170
+ headings = table.shift
171
+ table.unshift(col_widths.map { |col_width| '-' * col_width })
172
+ table.unshift(headings)
173
+ lines = table.map { |line| "| #{line.map { |i| i.to_s.gsub('\\', '\\\\').gsub('|', '\|') }.join(' | ')} |\n" }
174
+ lines[1] = lines[1].tr(' ', '-')
175
+ return lines.join.chomp
139
176
  end
140
177
  end
141
178
  end
data/lib/aspera/mime.rb CHANGED
@@ -9,8 +9,14 @@ module Aspera
9
9
  WWW = 'application/x-www-form-urlencoded'
10
10
  # Plain text body
11
11
  TEXT = 'text/plain'
12
+ # XML body
13
+ XML = 'application/xml'
14
+ # Multipart form body (file upload)
15
+ MULTIPART = 'multipart/form-data'
12
16
  # JSON:API body (https://jsonapi.org)
13
17
  JSON_API = 'application/vnd.api+json'
18
+ # Server-Sent Events stream
19
+ EVENT_STREAM = 'text/event-stream'
14
20
  # Check if a media type is JSON: `application/json` or structured syntax suffix `+json` (RFC 6839)
15
21
  # @param mime [String] Media type, without parameters (see `Rest.parse_header`)
16
22
  # @return [Boolean] `true` if JSON
data/lib/aspera/nagios.rb CHANGED
@@ -13,6 +13,8 @@ module Aspera
13
13
  DATE_WARN_OFFSET = 2
14
14
  DATE_CRIT_OFFSET = 5
15
15
  private_constant :LEVELS, :ADD_PREFIX, :DATE_WARN_OFFSET, :DATE_CRIT_OFFSET
16
+ # Exit code when the check itself fails
17
+ UNKNOWN = LEVELS.index(:unknown)
16
18
 
17
19
  # add methods to add nagios error levels, each take component name and message
18
20
  LEVELS.each_index do |code|
@@ -32,7 +34,7 @@ module Aspera
32
34
  # keep only errors in case of problem, other ok are assumed so
33
35
  data = res_errors unless res_errors.empty?
34
36
  # first is most critical
35
- data.sort! { |a, b| LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym) }
37
+ data = data.sort { |a, b| LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym) }
36
38
  # build message: if multiple components: concatenate
37
39
  # message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
38
40
  message = data
@@ -2,7 +2,6 @@
2
2
 
3
3
  # cspell:ignore precalc noxfer euid gettime
4
4
  require 'aspera/assert'
5
- require 'aspera/mime'
6
5
  require 'aspera/ascp/installation'
7
6
  require 'aspera/agent/transferd'
8
7
  require 'aspera/transfer/spec'
@@ -10,8 +9,8 @@ require 'aspera/transfer/error'
10
9
  require 'aspera/secret_hider'
11
10
  require 'aspera/string_ext'
12
11
  require 'aspera/log'
12
+ require 'aspera/servlet_helpers'
13
13
  require 'webrick'
14
- require 'openssl'
15
14
  require 'etc'
16
15
  require 'securerandom'
17
16
  require 'json'
@@ -319,59 +318,25 @@ module Aspera
319
318
  Aspera.assert(in_docroot?(File.realpath(folder_path))) { "Path outside docroot: #{path}" }
320
319
  Aspera.assert(File.directory?(folder_path)) { "Not a folder: #{path}" }
321
320
  folder_virtual = virtual_path(folder_path)
322
-
323
- # Build self structure
324
- folder_stat = File.stat(folder_path)
325
- structure = {
326
- 'self' => {
327
- 'path' => folder_virtual,
328
- 'basename' => File.basename(folder_path),
329
- 'type' => 'directory',
330
- # Node API reports 0 for folders
331
- 'size' => 0,
332
- 'mtime' => folder_stat.mtime.utc.iso8601,
333
- 'permissions' => permissions(folder_path)
334
- },
335
- 'items' => []
336
- }
337
-
338
- # Iterate over folder contents
339
- Dir.foreach(folder_path) do |entry|
340
- next if entry == '.' || entry == '..' # Skip current and parent directory
341
-
342
- item_path = File.join(folder_path, entry)
343
- item_type = File.ftype(item_path) rescue 'unknown' # Get the type of file
344
- item_type = FILE_TYPES.fetch(item_type, item_type)
345
- item_stat = File.lstat(item_path) # Use lstat to handle symbolic links correctly
346
-
347
- item = {
348
- 'path' => File.join(folder_virtual, entry),
349
- 'basename' => entry,
350
- 'type' => item_type,
351
- 'size' => item_type.eql?('directory') ? 0 : item_stat.size,
352
- 'mtime' => item_stat.mtime.utc.iso8601,
353
- 'permissions' => permissions(item_path)
354
- }
355
-
356
- # Add additional details for specific types
357
- case item_type
358
- when 'file'
359
- item['partial_file'] = false
360
- when 'symbolic_link'
361
- item['target'] = link_target(item_path)
362
- when 'unknown'
363
- item['note'] = 'File type could not be determined'
321
+ # stable order for paging: only the items of the page are read
322
+ names = Dir.children(folder_path).sort
323
+ items = names.drop(skip).first(count || names.length).map do |name|
324
+ item_path = File.join(folder_path, name)
325
+ # a symbolic link is not followed
326
+ item = item_info(item_path, File.join(folder_virtual, name), File.lstat(item_path))
327
+ case item['type']
328
+ when 'file' then item['partial_file'] = false
329
+ when 'symbolic_link' then item['target'] = link_target(item_path)
330
+ when 'unknown' then item['note'] = 'File type could not be determined'
364
331
  end
365
-
366
- structure['items'] << item
332
+ item
367
333
  end
368
-
369
- # stable order for paging
370
- all_items = structure['items'].sort_by { |item| item['basename'] }
371
- structure['items'] = all_items.drop(skip).first(count || all_items.length)
372
- structure['item_count'] = structure['items'].length
373
- structure['total_count'] = all_items.length
374
- structure
334
+ {
335
+ 'self' => item_info(folder_path, folder_virtual, File.stat(folder_path)),
336
+ 'items' => items,
337
+ 'item_count' => items.length,
338
+ 'total_count' => names.length
339
+ }
375
340
  end
376
341
 
377
342
  # Start a transfer, and monitor it in a thread until terminated
@@ -379,8 +344,8 @@ module Aspera
379
344
  # @return [String] transfer id
380
345
  def start(transfer_spec)
381
346
  Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
382
- # deep copy, returned in transfer `start_spec`
383
- start_spec = SecretHider.instance.deep_remove_secret(JSON.parse(transfer_spec.to_json))
347
+ # copy without secrets, returned in transfer `start_spec`
348
+ start_spec = SecretHider.instance.hidden_copy(transfer_spec)
384
349
  Log.dump(:start_spec, start_spec)
385
350
  request = ::Transferd::Api::TransferRequest.new(
386
351
  transferType: ::Transferd::Api::TransferType::FILE_REGULAR,
@@ -478,10 +443,23 @@ module Aspera
478
443
  def real_path(path)
479
444
  # `./` prevents expansion of `~`
480
445
  real = File.expand_path("./#{path.to_s.sub(%r{\A/+}, '')}", @docroot)
481
- Aspera.assert(in_docroot?(real)) { "Path outside docroot: #{path}" }
446
+ # symbolic links shall not lead out of the docroot
447
+ resolved = resolved_path(real) if in_docroot?(real)
448
+ Aspera.assert(!resolved.nil? && in_docroot?(resolved)) { "Path outside docroot: #{path}" }
482
449
  real
483
450
  end
484
451
 
452
+ # Resolve symbolic links of the existing part of a path: a destination may not exist yet
453
+ # @param real [String] absolute path, without `..`, within the docroot
454
+ # @return [String, nil] real path of the longest existing part of `real`, `nil` if it is a dangling symbolic link
455
+ def resolved_path(real)
456
+ # stops at the docroot, which exists
457
+ real = File.dirname(real) until File.exist?(real) || File.symlink?(real)
458
+ File.realpath(real)
459
+ rescue Errno::ENOENT, Errno::ELOOP
460
+ nil
461
+ end
462
+
485
463
  # Convert a real path to a Node API path
486
464
  # @param real [String] real path within the docroot
487
465
  # @return [String] path relative to the docroot, with leading `/`
@@ -496,6 +474,23 @@ module Aspera
496
474
  real.eql?(@docroot) || real.start_with?(@docroot.end_with?('/') ? @docroot : "#{@docroot}/")
497
475
  end
498
476
 
477
+ # Get the Node API fields of a folder or of an item of a folder
478
+ # @param real [String] real path within the docroot
479
+ # @param virtual [String] Node API path
480
+ # @param stat [File::Stat] information on the item
481
+ # @return [Hash] Node API fields common to a folder and its items
482
+ def item_info(real, virtual, stat)
483
+ {
484
+ 'path' => virtual,
485
+ 'basename' => File.basename(real),
486
+ 'type' => FILE_TYPES.fetch(stat.ftype, stat.ftype),
487
+ # Node API reports 0 for folders
488
+ 'size' => stat.directory? ? 0 : stat.size,
489
+ 'mtime' => stat.mtime.utc.iso8601,
490
+ 'permissions' => permissions(real)
491
+ }
492
+ end
493
+
499
494
  # Get the Node API permissions of an item
500
495
  # @param real [String] real path within the docroot
501
496
  # @return [Array<Hash>] Node API permissions, from the access of the emulator on the item
@@ -581,6 +576,8 @@ module Aspera
581
576
  # Answers a subset of the Node API, transfers are delegated to a NodeEmulator
582
577
  # a new instance is created for each request
583
578
  class NodeEmulatorServlet < WEBrick::HTTPServlet::AbstractServlet
579
+ include ServletHelpers
580
+
584
581
  PATH_TRANSFERS = '/ops/transfers'
585
582
  PATH_ONE_TRANSFER = %r{/ops/transfers/(.+)$}
586
583
  PATH_BROWSE = '/files/browse'
@@ -589,6 +586,8 @@ module Aspera
589
586
  CANCEL_STATUSES = %w[canceled cancelled stopped].freeze
590
587
  # `PUT` fields modified by transferd
591
588
  MODIFIABLE = %w[target_rate_kbps min_rate_kbps rate_policy].freeze
589
+ # HTTP methods with a JSON body
590
+ JSON_METHODS = %w[POST PUT].freeze
592
591
  # Create the servlet, with optional Basic authentication
593
592
  # @param server [WEBrick::HTTPServer]
594
593
  # @param config [Hash] `username` and `password` (Basic authentication expected from clients, optional)
@@ -599,14 +598,12 @@ module Aspera
599
598
  @expected_auth = "#{config[:username]}:#{config[:password]}" unless config[:username].nil?
600
599
  end
601
600
 
602
- # Check authentication, dispatch to `do_<verb>`, and send errors in Node API format
601
+ # Refuse requests of web pages, check authentication, dispatch to `do_<verb>`, and send errors in Node API format
603
602
  # @param request [WEBrick::HTTPRequest]
604
603
  # @param response [WEBrick::HTTPResponse]
605
604
  def service(request, response)
606
- unless authorized?(request)
607
- response['WWW-Authenticate'] = %Q(Basic realm="#{REALM}")
608
- return set_error(request, response, 401, 'Invalid or missing credentials')
609
- end
605
+ check_request(request, json: JSON_METHODS.include?(request.request_method))
606
+ return set_error(request, response, 401, 'Invalid or missing credentials', headers: {'WWW-Authenticate' => %Q(Basic realm="#{REALM}")}) unless authorized?(request)
610
607
  super
611
608
  rescue WEBrick::HTTPStatus::Error => e
612
609
  set_error(request, response, e.code, e.message)
@@ -623,40 +620,35 @@ module Aspera
623
620
  # @param request [WEBrick::HTTPRequest]
624
621
  # @param response [WEBrick::HTTPResponse]
625
622
  def do_POST(request, response)
626
- case request.path
627
- when PATH_TRANSFERS
628
- set_json_response(request, response, @simulator.transfer(@simulator.start(JSON.parse(request.body))))
629
- when PATH_BROWSE
630
- req = JSON.parse(request.body)
631
- set_json_response(request, response, @simulator.browse(req['path'] || '/', skip: req['skip'].to_i, count: req['count']&.to_i))
632
- else
633
- set_error(request, response, 404, "Unknown path: #{request.path}")
634
- end
623
+ json =
624
+ case request.path
625
+ when PATH_TRANSFERS
626
+ @simulator.transfer(@simulator.start(JSON.parse(request.body.to_s)))
627
+ when PATH_BROWSE
628
+ req = JSON.parse(request.body.to_s)
629
+ @simulator.browse(req['path'] || '/', skip: req['skip'].to_i, count: req['count']&.to_i)
630
+ else raise WEBrick::HTTPStatus::NotFound, "Unknown path: #{request.path}"
631
+ end
632
+ set_json_response(request, response, json)
635
633
  end
636
634
 
637
635
  # Get the node information, the list of transfers, or one transfer
638
636
  # @param request [WEBrick::HTTPRequest]
639
637
  # @param response [WEBrick::HTTPResponse]
640
638
  def do_GET(request, response)
641
- case request.path
642
- when '/info'
643
- set_json_response(request, response, @simulator.info)
644
- when PATH_TRANSFERS
645
- set_json_response(request, response, @simulator.transfers(
646
- active_only: query_boolean(request, 'active_only'),
647
- direction: request.query['direction']&.to_s,
648
- count: query_positive(request, 'count')
649
- ))
650
- when PATH_ONE_TRANSFER
651
- transfer = @simulator.transfer(request.path.match(PATH_ONE_TRANSFER)[1])
652
- if transfer.nil?
653
- set_error(request, response, 404, 'Unknown transfer')
654
- else
655
- set_json_response(request, response, transfer)
639
+ json =
640
+ case request.path
641
+ when '/info' then @simulator.info
642
+ when PATH_TRANSFERS
643
+ @simulator.transfers(
644
+ active_only: query_boolean(request, 'active_only'),
645
+ direction: request.query['direction']&.to_s,
646
+ count: query_positive(request, 'count')
647
+ )
648
+ when PATH_ONE_TRANSFER then known_transfer(@simulator.transfer(transfer_id(request)))
649
+ else raise WEBrick::HTTPStatus::NotFound, "Unknown path: #{request.path}"
656
650
  end
657
- else
658
- set_error(request, response, 404, "Unknown path: #{request.path}")
659
- end
651
+ set_json_response(request, response, json)
660
652
  end
661
653
 
662
654
  # Modify a transfer, or cancel it with `status`
@@ -676,7 +668,7 @@ module Aspera
676
668
  raise WEBrick::HTTPStatus::BadRequest, "Unsupported status: #{status}" unless CANCEL_STATUSES.include?(status)
677
669
  found = @simulator.cancel(id)
678
670
  end
679
- return set_error(request, response, 404, 'Unknown transfer') unless found
671
+ known_transfer(found)
680
672
  set_json_response(request, response, @simulator.transfer(id))
681
673
  end
682
674
 
@@ -684,7 +676,7 @@ module Aspera
684
676
  # @param request [WEBrick::HTTPRequest]
685
677
  # @param response [WEBrick::HTTPResponse]
686
678
  def do_CANCEL(request, response)
687
- return set_error(request, response, 404, 'Unknown transfer') unless @simulator.cancel(transfer_id(request))
679
+ known_transfer(@simulator.cancel(transfer_id(request)))
688
680
  response.status = 204
689
681
  end
690
682
 
@@ -700,6 +692,14 @@ module Aspera
700
692
  match[1]
701
693
  end
702
694
 
695
+ # @param found [Object] Result of the emulator for a transfer: `nil` or `false` if unknown
696
+ # @return [Object] found
697
+ # @raise [WEBrick::HTTPStatus::NotFound] if the transfer is unknown
698
+ def known_transfer(found)
699
+ raise WEBrick::HTTPStatus::NotFound, 'Unknown transfer' unless found
700
+ found
701
+ end
702
+
703
703
  # Read a boolean query parameter
704
704
  # @param request [WEBrick::HTTPRequest]
705
705
  # @param name [String] name of the query parameter
@@ -730,18 +730,16 @@ module Aspera
730
730
  # @param response [WEBrick::HTTPResponse]
731
731
  # @param code [Integer] HTTP status code
732
732
  # @param message [String] message for the user
733
- def set_error(request, response, code, message)
734
- set_json_response(request, response, {error: {code: code, reason: WEBrick::HTTPStatus.reason_phrase(code), user_message: message}}, code: code)
733
+ # @param headers [Hash{String => String}] Other headers of the response
734
+ def set_error(request, response, code, message, headers: {})
735
+ set_json_response(request, response, {error: {code: code, reason: WEBrick::HTTPStatus.reason_phrase(code), user_message: message}}, code: code, headers: headers)
735
736
  end
736
737
 
737
738
  # Check the Basic credentials of the request
738
739
  # @param request [WEBrick::HTTPRequest]
739
740
  # @return [Boolean] `true` if the request has the expected Basic credentials, or if none are expected
740
741
  def authorized?(request)
741
- return true if @expected_auth.nil?
742
- scheme, value = request['Authorization'].to_s.split(' ', 2)
743
- # constant time comparison
744
- scheme.to_s.casecmp?('Basic') && OpenSSL.secure_compare(value.to_s.unpack1('m'), @expected_auth)
742
+ @expected_auth.nil? || basic_authorized?(request, @expected_auth)
745
743
  end
746
744
 
747
745
  # Set the status and the JSON body of the response
@@ -749,11 +747,10 @@ module Aspera
749
747
  # @param response [WEBrick::HTTPResponse]
750
748
  # @param json [Hash, Array] body of the response
751
749
  # @param code [Integer] HTTP status code
752
- def set_json_response(request, response, json, code: 200)
753
- response.status = code
754
- response['Content-Type'] = Mime::JSON
755
- response.body = json.to_json
756
- Log.log.trace1 { Log.obj_dump("response for #{request.request_method} #{request.path}", json) }
750
+ # @param headers [Hash{String => String}] Other headers of the response
751
+ def set_json_response(request, response, json, code: 200, headers: {})
752
+ json_response(response, code, json, headers: headers)
753
+ Log.dump("response for #{request.request_method} #{request.path}", json, level: :trace1)
757
754
  end
758
755
  end
759
756
  end