aspera-cli 4.25.0.pre → 4.25.0

data/lib/aspera/preview/terminal.rb

@@ -3,11 +3,87 @@
 # cspell:words Magick MAGICKCORE ITERM mintty winsize termcap
 
 require 'rainbow'
+require 'base64'
 require 'io/console'
 require 'aspera/log'
 require 'aspera/environment'
+
 module Aspera
   module Preview
+    module Backend
+      # provides image pixels scaled to terminal
+      class Base
+        def initialize(reserve:, double:, font_ratio:)
+          @reserve = reserve
+          @height_ratio = double ? 2.0 : 1.0
+          @font_ratio = font_ratio
+        end
+        Aspera.require_method!(:terminal_pixels)
+        # compute scaling to fit terminal
+        def terminal_scaling(rows, columns)
+          (term_rows, term_columns) = IO.console.winsize || [24, 80]
+          term_rows = [term_rows - @reserve, 2].max
+          fit_term_ratio = [term_rows.to_f * @font_ratio / rows.to_f, term_columns.to_f / columns.to_f].min
+          [(columns * fit_term_ratio).to_i, (rows * fit_term_ratio * @height_ratio / @font_ratio).to_i]
+        end
+      end
+
+      class RMagick < Base
+        def initialize(blob, **kwargs)
+          super(**kwargs)
+          # do not require statically, as the package is optional
+          require 'rmagick' # https://rmagick.github.io/index.html
+          @image = Magick::ImageList.new.from_blob(blob)
+        end
+
+        def terminal_pixels
+          # quantum depth is 8 or 16, see: `magick xc: -format "%q" info:`
+          shift_for_8_bit = Magick::MAGICKCORE_QUANTUM_DEPTH - 8
+          # get all pixel colors, adjusted for Rainbow
+          pixel_colors = []
+          @image.scale(*terminal_scaling(@image.rows, @image.columns)).each_pixel do |pixel, col, row|
+            pixel_rgb = [pixel.red, pixel.green, pixel.blue]
+            pixel_rgb = pixel_rgb.map{ |color| color >> shift_for_8_bit} unless shift_for_8_bit.eql?(0)
+            # init 2-dim array
+            pixel_colors[row] ||= []
+            pixel_colors[row][col] = pixel_rgb
+          end
+          pixel_colors
+        end
+      end
+
+      class ChunkyPNG < Base
+        def initialize(blob, **kwargs)
+          super(**kwargs)
+          require 'chunky_png'
+          @png = ::ChunkyPNG::Image.from_blob(blob)
+        end
+
+        def terminal_pixels
+          src_w = @png.width
+          src_h = @png.height
+          dst_w, dst_h = terminal_scaling(src_h, src_w)
+          dst_w = [dst_w, 1].max
+          dst_h = [dst_h, 1].max
+          pixel_colors = Array.new(dst_h){Array.new(dst_w)}
+          x_ratio = src_w.to_f / dst_w
+          y_ratio = src_h.to_f / dst_h
+          dst_h.times do |dy|
+            sy = (dy * y_ratio).floor
+            sy = src_h - 1 if sy >= src_h
+            dst_w.times do |dx|
+              sx = (dx * x_ratio).floor
+              sx = src_w - 1 if sx >= src_w
+              rgba = @png.get_pixel(sx, sy)
+              # ChunkyPNG stores as 0xRRGGBBAA; extract 8-bit channels
+              pixel_colors[dy][dx] = %i[r g b].map{ |i| ::ChunkyPNG::Color.send(i, rgba)}
+            end
+          end
+          pixel_colors
+        end
+      end
+    end
+
     # Display a picture in the terminal.
     # Either use coloured characters or iTerm2 protocol.
     class Terminal
@@ -22,41 +98,31 @@ module Aspera
       private_constant :TERM_ENV_VARS, :ITERM_NAMES, :DEFAULT_FONT_RATIO
       class << self
         # @param blob       [String]  The image as a binary string
-        # @param reserve    [Integer] Number of lines to reserve for other text than the image
         # @param text       [Boolean] `true` to display the image as text, `false` to use iTerm2 if supported
+        # @param reserve    [Integer] Number of lines to reserve for other text than the image
         # @param double     [Boolean] `true` to use colors on half lines, `false` to use colors on full lines
         # @param font_ratio [Float]   ratio = font height / font width
         # @return [String] The image as text, or the iTerm2 escape sequence
-        def build(blob, reserve: 3, text: false, double: true, font_ratio: DEFAULT_FONT_RATIO)
+        def build(blob, text: false, reserve: 3, double: true, font_ratio: DEFAULT_FONT_RATIO)
           return '[Image display requires a terminal]' unless Environment.terminal?
           return iterm_display_image(blob) if iterm_supported? && !text
-          begin
-            # do not require statically, as the package is optional
-            require 'rmagick' # https://rmagick.github.io/index.html
-          rescue LoadError => e
-            Log.log.error('Install missing gem: gem install rmagick')
-            # fallback to iterm, if supported
+          pixel_colors =
+            begin
+              Log.log.debug('Trying chunky_png')
+              Backend::ChunkyPNG.new(blob, reserve: reserve, double: double, font_ratio: font_ratio).terminal_pixels
+            rescue => e
+              Log.log.debug(e.message)
+              begin
+                Log.log.debug('Trying rmagick')
+                Backend::RMagick.new(blob, reserve: reserve, double: double, font_ratio: font_ratio).terminal_pixels
+              rescue => e
+                Log.log.debug(e.message)
+                nil
+              end
+            end
+          if pixel_colors.nil?
             return iterm_display_image(blob) if iterm_supported?
-            Log.log.error('Cant display picture.')
-            raise e
-          end
-          image = Magick::ImageList.new.from_blob(blob)
-          (term_rows, term_columns) = IO.console.winsize
-          term_rows -= reserve
-          # compute scaling to fit terminal
-          fit_term_ratio = [term_rows.to_f * font_ratio / image.rows.to_f, term_columns.to_f / image.columns.to_f].min
-          height_ratio = double ? 2.0 : 1.0
-          image = image.scale((image.columns * fit_term_ratio).to_i, (image.rows * fit_term_ratio * height_ratio / font_ratio).to_i)
-          # quantum depth is 8 or 16, see: `magick xc: -format "%q" info:`
-          shift_for_8_bit = Magick::MAGICKCORE_QUANTUM_DEPTH - 8
-          # get all pixel colors, adjusted for Rainbow
-          pixel_colors = []
-          image.each_pixel do |pixel, col, row|
-            pixel_rgb = [pixel.red, pixel.green, pixel.blue]
-            pixel_rgb = pixel_rgb.map{ |color| color >> shift_for_8_bit} unless shift_for_8_bit.eql?(0)
-            # init 2-dim array
-            pixel_colors[row] ||= []
-            pixel_colors[row][col] = pixel_rgb
+            raise 'Cannot decode picture.'
           end
           # now generate text
           text_pixels = []
@@ -88,7 +154,7 @@ module Aspera
           }.map{ |k, v| "#{k}=#{v}"}.join(';')
           # \a is BEL, \e is ESC : https://github.com/ruby/ruby/blob/master/doc/syntax/literals.rdoc#label-Strings
           # escape sequence for iTerm2 image display
-          return "\e]1337;File=#{arguments}:#{Base64.encode64(blob)}\a"
+          return "\e]1337;File=#{arguments}:#{Base64.strict_encode64(blob)}\a"
         end
 
         # @return [Boolean] true if the terminal supports iTerm2 image display

data/lib/aspera/preview/utils.rb

@@ -44,17 +44,18 @@ module Aspera
           end
         end
 
-        # execute external command
-        # one could use "system", but we would need to redirect stdout/err
-        # @return nil
+        # Execute external command
+        # @return [nil]
         def external_command(command_sym, command_args)
           Aspera.assert_values(command_sym, EXTERNAL_TOOLS){'command'}
-          Environment.secure_execute(exec: command_sym.to_s, args: command_args.map(&:to_s), out: File::NULL, err: File::NULL)
+          Environment.secure_execute(command_sym.to_s, *command_args.map(&:to_s), out: File::NULL, err: File::NULL)
         end
 
+        # Execute external command and capture output
+        # @return [String]
         def external_capture(command_sym, command_args)
           Aspera.assert_values(command_sym, EXTERNAL_TOOLS){'command'}
-          return Environment.secure_capture(exec: command_sym.to_s, args: command_args.map(&:to_s))
+          return Environment.secure_execute(command_sym.to_s, *command_args.map(&:to_s), mode: :capture)
         end
 
         def ffmpeg(gl_p: FFMPEG_DEFAULT_PARAMS, in_p: [], in_f:, out_p: [], out_f:)

data/lib/aspera/rest.rb

@@ -307,9 +307,12 @@ module Aspera
     # @param body       [Hash, String] body parameters
     # @param headers      [Hash] additional headers (override Content-Type)
     # @param save_to_file [String, nil](filepath)
-    # @param exception    [Bool] `true`, error raise exception
+    # @param exception    [Boolean] `true`, error raise exception
     # @param ret [:data, :resp, :both] Tell to return only data, only http response, or both
-    # @return [Object, Array] only data, only http response, or both
+    # @return [(HTTPResponse,Hash)] If ret is :both
+    # @return [HTTPResponse] If ret is :resp
+    # @return [Hash] If ret is :data
+    # @raise [RestCallError] on error if `exception` is true
     def call(
       operation:,
       subpath: nil,

data/lib/aspera/ssh.rb

@@ -35,16 +35,17 @@ module Aspera
     # ssh_options: same as Net::SSH.start
     # see: https://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start
     def initialize(host, username, ssh_options)
-      Log.log.debug{"ssh:#{username}@#{host}"}
-      Log.log.debug{"ssh_options:#{ssh_options}"}
       Aspera.assert_type(host, String)
       Aspera.assert_type(username, String)
       Aspera.assert_type(ssh_options, Hash)
-      ssh_options[:use_agent] = false unless ssh_options.key?(:use_agent)
       @host = host
       @username = username
-      @ssh_options = ssh_options
-      @ssh_options[:logger] = Log.log
+      @ssh_options = ssh_options.dup
+      @ssh_options[:logger] = Log.log unless @ssh_options.key?(:logger)
+      @ssh_options[:verbose] = :warn unless @ssh_options.key?(:verbose)
+      @ssh_options[:use_agent] = false unless @ssh_options.key?(:use_agent)
+      Log.log.debug{"ssh:#{@username}@#{@host}"}
+      Log.dump(:ssh_options, @ssh_options)
     end
 
     # Anything on stderr raises an exception

data/lib/aspera/sync/conf.schema.yaml

@@ -4,8 +4,8 @@ description: Async session description, identical to `asyncs` Node API.
 $comment: >-
   This is a YAML version of the original JSON schema from asyncs API.
   It is modified with special `x-` fields.
-  x-ts-name      [Bool,String]  `true` if same name in transfer spec, else real name in transfer spec
-  x-ts-convert   [String]       Name of methods to convert value from transfer spec to `conf`
+  x-ts-name      [Boolean,String] `true` if same name in transfer spec, else real name in transfer spec
+  x-ts-convert   [String]         Name of methods to convert value from transfer spec to `conf`
   `x-` fields are also documented in `command_line_builder.rb`
 type: object
 required:

data/lib/aspera/sync/operations.rb

@@ -146,7 +146,7 @@ module Aspera
               session_builder.process_params
               session_builder.add_env_args(env_args)
             end
-            Environment.secure_execute(exec: Ascp::Installation.instance.path(:async), **env_args)
+            Environment.secure_execute(Ascp::Installation.instance.path(:async), *env_args[:args], env: env_args[:env])
           end
           return
         end
@@ -174,7 +174,7 @@ module Aspera
         # @return [Hash] parsed output of asyncadmin
         def admin_status(sync_info)
           Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
-          arguments = ['--quiet']
+          arguments = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
           if sync_info.key?('local')
             # `conf` format
             arguments.push("--name=#{sync_info['name']}")
@@ -197,7 +197,7 @@ module Aspera
               raise Error, 'Missing either local_db_dir or local_dir'
             end
           end
-          stdout = Environment.secure_capture(exec: ASYNC_ADMIN_EXECUTABLE, args: arguments)
+          stdout = Environment.secure_execute(*arguments, mode: :capture)
           return parse_status(stdout)
         end
 

data/lib/aspera/transfer/parameters.rb

@@ -45,12 +45,12 @@ module Aspera
         end
       end
 
-      # @param job_spec        [Hash]   Transfer spec
-      # @param ascp_args       [Array]  Other ascp args
-      # @param quiet           [Bool]   Remove ascp output
-      # @param trusted_certs   [Array]  Trusted certificates
-      # @param client_ssh_key  [Symbol] :rsa or :dsa
-      # @param check_ignore_cb [Proc]   Callback
+      # @param job_spec        [Hash]    Transfer spec
+      # @param ascp_args       [Array]   Other ascp args
+      # @param quiet           [Boolean] Remove ascp output
+      # @param trusted_certs   [Array]   Trusted certificates
+      # @param client_ssh_key  [:rsa,:dsa] :rsa or :dsa
+      # @param check_ignore_cb [Proc]    Callback
       def initialize(
         job_spec,
         ascp_args:       nil,

data/lib/aspera/transfer/spec.schema.yaml

@@ -7,8 +7,8 @@ $comment: >-
   The following extensions are used for mapping to ascp arguments.
   x-cli-envvar   [String]       Name of env var
   x-cli-option   [String]       Command line option (starts with "-"), or `true`: same as ts, or `false`: not an option
-  x-cli-switch   [Bool]         `true` if option has no arg, else by default option has a value
-  x-cli-special  [Bool]         `true` if not anaged by command line generator (special handling: option or argument)
+  x-cli-switch   [Boolean]      `true` if option has no arg, else by default option has a value
+  x-cli-special  [Boolean]      `true` if not anaged by command line generator (special handling: option or argument)
   x-cli-convert  [String,Hash]  Method name for Convert object or Conversion Hash for enum: ts to arg
   x-agents       [Array]        Supported agents (for doc only), if not specified: all
   x-deprecation  [String]       Deprecation message for doc
@@ -106,7 +106,7 @@ properties:
   delete_source:
     $comment: "TODO: implement"
     description: >-
-      Remove transfered source files after transfer success.
+      Remove transferred source files after transfer success.
       Equivalent to `remove_after_transfer` + `remove_empty_directories` + `remove_empty_source_directory`.
       Take precedence over those.
     type: boolean
@@ -881,7 +881,7 @@ properties:
   xfer_max_retries:
     description: >-
       Maximum number of retries, for node API initiated transfers.
-      Shall not exceed aspera.conf `transfer_manager_max_retries` (default 5).
+      Shall not exceed `aspera.conf` parameter `transfer_manager_max_retries` (default 5).
     type: integer
     x-agents:
       - node

data/lib/aspera/transfer/spec_doc.rb

@@ -8,20 +8,14 @@ module Aspera
     # Generate documentation from Schema, for Transfer Spec, or async Conf spec
     class SpecDoc
       class << self
-        # First letter of agent name symbol
-        def agent_to_short(agent_sym)
-          agent_sym.to_sym.eql?(:direct) ? :a : agent_sym.to_s[0].to_sym
-        end
-
-        # @param formatter      [Cli::Formatter] Formatter to use, methods: markdown, tick, check_row
+        # @param formatter      [Cli::Formatter] Formatter to use, methods: markdown_text, tick, check_row
         # @param include_option [Boolean]        `true` : include CLI options
         # @param agent_columns  [Boolean]        `true` : include agents columns
         # @param schema         [Hash]           The JSON spec
         # @return [Array] a table suitable to display in manual
         def man_table(formatter, include_option: false, agent_columns: true, schema: Spec::SCHEMA)
-          col_local = agent_to_short(:direct)
           cols = %i[name type description]
-          cols.insert(-2, *AGENT_LIST.map(&:last)) if agent_columns
+          cols.insert(-2, *Agent::Factory::ALL.values.map{ |i| i[:short]}.sort) if agent_columns
           rows = []
           schema['properties'].each do |name, info|
             rows.concat(man_table(formatter, include_option: include_option, agent_columns: agent_columns, schema: info).last.map{ |h| h.merge(name: "#{name}.#{h[:name]}")}) if info['type'].eql?('object') && info['properties']
@@ -35,25 +29,25 @@ module Aspera
             # Render Markdown formatting and split lines
             columns[:description] =
               info['description']
-                .gsub(Markdown::FORMATS){formatter.markdown(Regexp.last_match)}
+                .gsub(Markdown::FORMATS){formatter.markdown_text(Regexp.last_match)}
                 .split("\n") if info.key?('description')
             columns[:description].unshift("DEPRECATED: #{info['x-deprecation']}") if info.key?('x-deprecation')
             # Add flags for supported agents in doc
             agents = []
-            AGENT_LIST.each do |agent_info|
-              agents.push(agent_info.last) if info['x-agents'].nil? || info['x-agents'].include?(agent_info.first.to_s)
+            Agent::Factory::ALL.each_key do |sym|
+              agents.push(sym) if info['x-agents'].nil? || info['x-agents'].include?(sym.to_s)
             end
-            Aspera.assert(agents.include?(col_local)){"#{name}: x-cli-option requires agent direct (or nil)"} if info['x-cli-option']
+            Aspera.assert(agents.include?(:direct)){"#{name}: x-cli-option requires agent direct (or nil)"} if info['x-cli-option']
             if agent_columns
-              AGENT_LIST.each do |agent_info|
-                columns[agent_info.last] = formatter.tick(agents.include?(agent_info.last))
+              Agent::Factory::ALL.each do |sym, names|
+                columns[names[:short]] = formatter.tick(agents.include?(sym))
               end
             else
-              columns[:description].push("(#{agents.map(&:upcase).join(', ')})") unless agents.length.eql?(AGENT_LIST.length)
+              columns[:description].push("(#{agents.map{ |i| Agent::Factory::ALL[i][:short].to_s.upcase}.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
             end
             # Only keep lines that are usable in supported agents
             next false if agents.empty?
-            columns[:description].push("Allowed values: #{info['enum'].map{ |v| formatter.markdown("`#{v}`")}.join(', ')}") if info.key?('enum')
+            columns[:description].push("Allowed values: #{info['enum'].map{ |v| formatter.markdown_text("`#{v}`")}.join(', ')}") if info.key?('enum')
             if include_option
               envvar_prefix = ''
               cli_option =
@@ -70,17 +64,13 @@ module Aspera
                   "#{info['x-cli-option']}#{sep}#{"(#{conversion_tag})" if conversion_tag}#{arg_type}"
                 end
               short = info.key?('x-cli-short') ? "(#{info['x-cli-short']})" : nil
-              columns[:description].push("(#{'special:' if info['x-cli-special']}#{envvar_prefix}#{formatter.markdown("`#{cli_option}`")})#{short}") if cli_option
+              columns[:description].push("(#{'special:' if info['x-cli-special']}#{envvar_prefix}#{formatter.markdown_text("`#{cli_option}`")})#{short}") if cli_option
             end
             rows.push(formatter.check_row(columns))
           end
           [cols, rows.sort_by{ |i| i[:name]}]
         end
       end
-      # Agents shown in manual for parameters (sub list)
-      AGENT_LIST = Agent::Factory.instance.list.map do |agent_sym|
-        [agent_sym, agent_sym.to_s.capitalize, agent_to_short(agent_sym)]
-      end.sort_by(&:last).freeze
     end
   end
 end

data/lib/aspera/uri_reader.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'uri'
+require 'base64'
 require 'aspera/assert'
 require 'aspera/rest'
 require 'aspera/temp_file_manager'
@@ -19,6 +20,13 @@ module Aspera
         case uri.scheme
         when 'http', 'https'
           return Rest.new(base_url: uri_to_read, redirect_max: 5).read(nil, headers: {'Accept' => '*/*'})
+        when 'data'
+          metadata, encoded_data = uri.opaque.split(',', 2)
+          if metadata.end_with?(';base64')
+            Base64.decode64(encoded_data)
+          else
+            URI.decode_www_form_component(encoded_data)
+          end
         when SCHEME_FILE, NilClass
           local_file_path = uri.path
           raise Error, 'URL shall have a path, check syntax' if local_file_path.nil?
@@ -35,12 +43,18 @@ module Aspera
           # require specific file scheme: the path part is "relative", or absolute if there are 4 slash
           raise "use format: #{SCHEME_FILE_PFX2}<path>" unless url.start_with?(SCHEME_FILE_PFX2)
           return File.expand_path(url[SCHEME_FILE_PFX2.length..-1])
+        elsif url.start_with?('data:')
+          # download to temp file
+          # auto-delete on exit
+          temp_file = TempFileManager.instance.new_file_path_global('uri_reader')
+          File.write(temp_file, read(url), binmode: true)
+          return temp_file
         else
           # download to temp file
           # auto-delete on exit
-          sdk_archive_path = TempFileManager.instance.new_file_path_global(suffix: File.basename(url))
-          Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to_file: sdk_archive_path)
-          return sdk_archive_path
+          temp_file = TempFileManager.instance.new_file_path_global(suffix: File.basename(url))
+          Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to_file: temp_file)
+          return temp_file
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aspera-cli
 version: !ruby/object:Gem::Version
-  version: 4.25.0.pre
+  version: 4.25.0
 platform: ruby
 authors:
 - Laurent Martin
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: chunky_png
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement
@@ -353,6 +367,7 @@ files:
 - lib/aspera/data/5
 - lib/aspera/data/6
 - lib/aspera/data_repository.rb
+- lib/aspera/dot_container.rb
 - lib/aspera/environment.rb
 - lib/aspera/faspex_gw.rb
 - lib/aspera/faspex_postproc.rb
@@ -442,7 +457,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements:
 - Read the manual for any requirement
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: 'Execute actions using command line on IBM Aspera Server products: Aspera
   on Cloud, Faspex, Shares, Node, Console, Orchestrator, High Speed Transfer Server'