aspera-cli 4.14.0 → 4.16.0

data/lib/aspera/ascmd.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+# cspell:ignore ascmd smode errstr zstr zmode zuid zgid zctime zatime zmtime fcount dcount btype blist codeset lc_ctype ascmdtypes
 require 'aspera/log'
+require 'aspera/assert'
 
 module Aspera
   # Run +ascmd+ commands using specified executor (usually, remotely on transfer node)
@@ -20,9 +22,16 @@ module Aspera
     # @param [Symbol] one of OPERATIONS
     # @param [Array] parameters for "as" command
     # @return result of command, type depends on command (bool, array, hash)
-    def execute_single(action_sym, args=nil)
-      # concatenate arguments, enclose in double quotes, protect backslash and double quotes, add "as_" command and as_exit
-      stdin_input = (args || []).map{|v| '"' + v.gsub(/["\\]/n) {|s| '\\' + s } + '"'}.unshift('as_' + action_sym.to_s).join(' ') + "\nas_exit\n"
+    def execute_single(action_sym, arguments=nil)
+      # add "as_" command
+      main_command = ["as_#{action_sym}"]
+      arguments&.each do |v|
+        # enclose arguments in double quotes, protect backslash and double quotes
+        main_command.push(%Q{"#{v.gsub(/["\\]/n){|s|"\\#{s}"}}"})
+      end
+      # execute the main command and then exit
+      stdin_input = [main_command.join(' '), 'as_exit', ''].join("\n")
+      Log.log.debug{"execute_single:#{stdin_input}"}
       # execute, get binary output
       byte_buffer = @command_executor.execute('ascmd', stdin_input).unpack('C*')
       raise 'ERROR: empty answer from server' if byte_buffer.empty?
@@ -44,29 +53,27 @@ module Aspera
         end
       end
       # for info, second overrides first, so restore it
-      case result.keys.length; when 0 then result = system_info; when 1 then result = result[result.keys.first]; else raise 'error'; end
+      case result.keys.length; when 0 then result = system_info; when 1 then result = result[result.keys.first]; else error_unexpected_value(result.keys.length); end
       # raise error as exception
-      raise Error.new(result[:errno], result[:errstr], action_sym, args) if
+      raise Error.new(result[:errno], result[:errstr], action_sym, arguments) if
         result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{|i|i[:name]}.sort)
       return result
     end # execute_single
 
     # This exception is raised when +ascmd+ returns an error.
     class Error < StandardError
-      attr_reader :errno, :errstr, :command, :args
-
-      def initialize(errno, errstr, cmd, args); super(); @errno = errno; @errstr = errstr; @command = cmd; @args = args; end # rubocop:disable Style/Semicolon
-
-      def message; "ascmd: (#{errno}) #{errstr}"; end
+      def initialize(errno, errstr, cmd, arguments)
+        super(); @errno = errno; @errstr = errstr; @command = cmd; @arguments = arguments; end # rubocop:disable Style/Semicolon
 
-      def extended_message; "ascmd: errno=#{errno} errstr=\"#{errstr}\" command=\"#{command}\" args=#{args}"; end
+      def message; "ascmd: #{@errstr} (#{@errno})"; end
+      def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments.join(',')}"; end
     end # Error
 
     # description of result structures (see ascmdtypes.h). Base types are big endian
     # key = name of type
     TYPES_DESCR = {
       result: {decode: :field_list,
-               fields: [{name: :file, is_a: :stat}, {name: :dir, is_a: :stat, special: :substruct}, {name: :size, is_a: :size}, {name: :error, is_a: :error},
+               fields: [{name: :file, is_a: :stat}, {name: :dir, is_a: :stat, special: :sub_struct}, {name: :size, is_a: :size}, {name: :error, is_a: :error},
                         {name: :info, is_a: :info}, {name: :success, is_a: nil, special: :return_true}, {name: :exit, is_a: nil},
                         {name: :df, is_a: :mnt, special: :restart_on_first}, {name: :md5sum, is_a: :md5sum}]},
       stat:   {decode: :field_list,
@@ -118,7 +125,7 @@ module Aspera
         indent_level = (indent_level || -1) + 1
         type_descr = TYPES_DESCR[type_name]
         raise "Unexpected type #{type_name}" if type_descr.nil?
-        Log.log.debug{"#{'   .' * indent_level}parse:#{type_name}:#{type_descr[:decode]}:#{buffer[0, 16]}...".red}
+        Log.log.trace1{"#{'   .' * indent_level}parse:#{type_name}:#{type_descr[:decode]}:#{buffer[0, 16]}...".red}
         result = nil
         case type_descr[:decode]
         when :base
@@ -128,7 +135,7 @@ module Aspera
           byte_array = [byte_array] unless byte_array.is_a?(Array)
           result = byte_array.pack('C*').unpack1(type_descr[:unpack])
           result.force_encoding('UTF-8') if type_name.eql?(:zstr)
-          Log.log.debug{"#{'   .' * indent_level}-> base:#{byte_array} -> #{result}"}
+          Log.log.trace1{"#{'   .' * indent_level}-> base:#{byte_array} -> #{result}"}
           result = Time.at(result) if type_name.eql?(:epoch)
         when :buffer_list
           result = []
@@ -138,7 +145,7 @@ module Aspera
             raise 'ERROR:not enough bytes' if buffer.length < length
             value = buffer.shift(length)
             result.push({btype: btype, buffer: value})
-            Log.log.debug{"#{'   .' * indent_level}:buffer_list[#{result.length - 1}] #{result.last}"}
+            Log.log.trace1{"#{'   .' * indent_level}:buffer_list[#{result.length - 1}] #{result.last}"}
           end
         when :field_list
           # by default the result is one struct
@@ -147,13 +154,13 @@ module Aspera
           parse(buffer, :blist, indent_level).each do |typed_buffer|
             # what type of field is it ?
             field_info = field_description(type_name, typed_buffer)
-            Log.log.debug{"#{'   .' * indent_level}+ field(special=#{field_info[:special]})=#{field_info[:name]}".green}
+            Log.log.trace1{"#{'   .' * indent_level}+ field(special=#{field_info[:special]})=#{field_info[:name]}".green}
             case field_info[:special]
             when nil
               result[field_info[:name]] = parse(typed_buffer[:buffer], field_info[:is_a], indent_level)
             when :return_true
               result[field_info[:name]] = true
-            when :substruct
+            when :sub_struct
               result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{|r|parse(r[:buffer], field_info[:is_a], indent_level)}
             when :multiple
               result[field_info[:name]] ||= []
@@ -167,7 +174,7 @@ module Aspera
               end
             end
           end
-        else raise "error: unknown decode:#{type_descr[:decode]}"
+        else error_unexpected_value(type_descr[:decode])
         end # is_a
         return result
       end

data/lib/aspera/assert.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Aspera
+  class InternalError < StandardError
+  end
+
+  class AssertError < StandardError
+  end
+end
+
+class Object
+  def assert(assertion, info = nil, level: 2, exception_class: Aspera::AssertError)
+    raise Aspera::InternalError, 'bad assert: both info and block given' unless info.nil? || !block_given?
+    return if assertion
+    message = 'assertion failed'
+    info = yield if block_given?
+    message = "#{message}: #{info}" if info
+    message = "#{message}: #{caller(level..level).first}"
+    raise exception_class, message
+  end
+
+  # assert that value has the given type
+  # @param value [Object] the value to check
+  # @param type [Class] the expected type
+  def assert_type(value, type, exception_class: Aspera::AssertError)
+    assert(value.is_a?(type), level: 3, exception_class: exception_class){"#{block_given? ? "#{yield}: " : nil}expecting #{type}, but have #{value.inspect}"}
+  end
+
+  # assert that value is one of the given values
+  def assert_values(value, values, exception_class: Aspera::AssertError)
+    assert(values.include?(value), level: 3, exception_class: exception_class) do
+      "#{block_given? ? "#{yield}: " : nil}expecting one of #{values.inspect}, but have #{value.inspect}"
+    end
+  end
+
+  # the line with this shall never be reached
+  def error_unreachable_line
+    raise Aspera::InternalError, "unreachable line reached: #{caller(2..2).first}"
+  end
+
+  # the value is not one of the expected values
+  def error_unexpected_value(value, exception_class: Aspera::InternalError)
+    raise exception_class, "#{block_given? ? "#{yield}: " : nil}unexpected value: #{value.inspect}"
+  end
+end

data/lib/aspera/cli/basic_auth_plugin.rb

@@ -8,17 +8,20 @@ module Aspera
     # base class for applications supporting basic authentication
     class BasicAuthPlugin < Aspera::Cli::Plugin
       class << self
-        def register_options(env)
-          env[:options].declare(:url, 'URL of application, e.g. https://org.asperafiles.com')
-          env[:options].declare(:username, 'Username to log in')
-          env[:options].declare(:password, "User's password")
-          env[:options].parse_options!
+        @@basic_options_declared = false # rubocop:disable Style/ClassVars
+        def declare_options(options, force: false)
+          return if @@basic_options_declared && !force
+          @@basic_options_declared = true # rubocop:disable Style/ClassVars
+          options.declare(:url, 'URL of application, e.g. https://faspex.example.com/aspera/faspex')
+          options.declare(:username, 'Username to log in')
+          options.declare(:password, "User's password")
+          options.parse_options!
         end
       end
 
       def initialize(env)
         super(env)
-        self.class.register_options(env) unless env[:skip_basic_auth_options]
+        BasicAuthPlugin.declare_options(options, force: env[:all_manuals])
       end
 
       # returns a Rest object with basic auth

data/lib/aspera/cli/error.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Aspera
+  module Cli
+    # CLI base exception
+    class Error < StandardError; end
+
+    # raised when an unexpected argument is provided
+    class BadArgument < Error; end
+
+    class NoSuchIdentifier < Error
+      def initialize(res_type, res_id)
+        super("#{res_type} with identifier #{res_id} not found")
+      end
+    end
+  end
+end

data/lib/aspera/cli/extended_value.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
+# cspell:ignore csvt jsonpp
 require 'aspera/uri_reader'
 require 'aspera/environment'
+require 'aspera/log'
+require 'aspera/assert'
 require 'json'
 require 'base64'
 require 'zlib'
@@ -14,6 +17,11 @@ module Aspera
     class ExtendedValue
       include Singleton
 
+      # special values
+      INIT = 'INIT'
+      ALL = 'ALL'
+      DEF = 'DEF'
+
       class << self
         # decode comma separated table text
         def decode_csvt(value)
@@ -24,35 +32,42 @@ module Aspera
             if col_titles.nil?
               col_titles = values
             else
-              entry = {}
-              col_titles.each{|title|entry[title] = values.shift}
-              hash_array.push(entry)
+              hash_array.push(col_titles.zip(values).to_h)
             end
           end
           Log.log.warn('Titled CSV file without any line') if hash_array.empty?
           return hash_array
         end
+
+        def assert_no_value(v, what)
+          raise "no value allowed for extended value type: #{what}" unless v.empty?
+        end
       end
 
       private
 
       def initialize
+        # base handlers
+        # other handlers can be set using set_handler, e.g. `preset` is reader in config plugin
         @handlers = {
+          val:    lambda{|v|v},
           base64: lambda{|v|Base64.decode64(v)},
           csvt:   lambda{|v|ExtendedValue.decode_csvt(v)},
-          env:    lambda{|v|ENV[v]},
+          env:    lambda{|v|ENV.fetch(v, nil)},
           file:   lambda{|v|File.read(File.expand_path(v))},
+          uri:    lambda{|v|UriReader.read(v)},
           json:   lambda{|v|JSON.parse(v)},
           lines:  lambda{|v|v.split("\n")},
           list:   lambda{|v|v[1..-1].split(v[0])},
+          none:   lambda{|v|ExtendedValue.assert_no_value(v, :none); nil}, # rubocop:disable Style/Semicolon
           path:   lambda{|v|File.expand_path(v)},
-          ruby:   lambda{|v|Environment.secure_eval(v)},
-          secret: lambda{|v|raise 'no value allowed for secret' unless v.empty?; $stdin.getpass('secret> ')}, # rubocop:disable Style/Semicolon
-          stdin:  lambda{|v|raise 'no value allowed for stdin' unless v.empty?; $stdin.read}, # rubocop:disable Style/Semicolon
-          uri:    lambda{|v|UriReader.read(v)},
-          val:    lambda{|v|v},
-          zlib:   lambda{|v|Zlib::Inflate.inflate(v)}
-          # other handlers can be set using set_handler, e.g. preset is reader in config plugin
+          re:     lambda{|v|Regexp.new(v)},
+          ruby:   lambda{|v|Environment.secure_eval(v, __FILE__, __LINE__)},
+          secret: lambda{|v|prompt = v.empty? ? 'secret' : v; $stdin.getpass("#{prompt}> ")}, # rubocop:disable Style/Semicolon
+          stdin:  lambda{|v|ExtendedValue.assert_no_value(v, :stdin); $stdin.read}, # rubocop:disable Style/Semicolon
+          yaml:   lambda{|v|YAML.load(v)},
+          zlib:   lambda{|v|Zlib::Inflate.inflate(v)},
+          extend: lambda{|v|ExtendedValue.instance.evaluate_all(v)}
         }
       end
 
@@ -63,25 +78,45 @@ module Aspera
       # add a new handler
       def set_handler(name, method)
         Log.log.debug{"setting handler for #{name}"}
-        raise 'name must be Symbol' unless name.is_a?(Symbol)
+        assert_type(name, Symbol){'name'}
         @handlers[name] = method
       end
 
+      # Regex to match an extended value
+      def ext_re
+        "@(#{modifiers.join('|')}):"
+      end
+
       # parse an option value if it is a String using supported extended value modifiers
       # other value types are returned as is
       def evaluate(value)
-        return value if !value.is_a?(String)
+        return value unless value.is_a?(String)
+        regex = Regexp.new("^#{ext_re}(.*)$")
         # first determine decoders, in reversed order
         handlers_reversed = []
-        while (m = value.match(/^@([^:]+):(.*)/)) && @handlers.include?(m[1].to_sym)
-          handlers_reversed.unshift(m[1].to_sym)
+        while (m = value.match(regex))
+          handler = m[1].to_sym
+          handlers_reversed.unshift(handler)
           value = m[2]
+          # stop processing if handler is extend (it will be processed later)
+          break if handler.eql?(:extend)
         end
         handlers_reversed.each do |handler|
           value = @handlers[handler].call(value)
         end
         return value
-      end # parse
+      end # evaluate
+
+      # find inner extended values
+      def evaluate_all(value)
+        regex = Regexp.new("^(.*)#{ext_re}([^@]*)@(.*)$")
+        while (m = value.match(regex))
+          sub_value = "@#{m[2]}:#{m[3]}"
+          Log.log.debug("evaluating #{sub_value}")
+          value = m[1] + evaluate(sub_value) + m[4]
+        end
+        return value
+      end
     end
   end
 end