aspera-cli 4.19.0 → 4.21.1

data/lib/aspera/ascp/installation.rb

@@ -3,24 +3,30 @@
 # cspell:ignore protobuf ckpt
 require 'aspera/environment'
 require 'aspera/data_repository'
-require 'aspera/ascp/products'
 require 'aspera/log'
+require 'aspera/rest'
+require 'aspera/uri_reader'
 require 'aspera/assert'
 require 'aspera/web_server_simple'
+require 'aspera/cli/info'
+require 'aspera/cli/version'
+require 'aspera/products/alpha'
+require 'aspera/products/connect'
+require 'aspera/products/transferd'
+require 'aspera/products/other'
 require 'English'
 require 'singleton'
 require 'xmlsimple'
-require 'zlib'
 require 'base64'
 require 'fileutils'
 require 'openssl'
-
+require 'yaml'
 module Aspera
   module Ascp
     # Singleton that tells where to find ascp and other local resources (keys..) , using the "path(:name)" method.
     # It is used by object : AgentDirect to find necessary resources
     # By default it takes the first Aspera product found
-    # but the user can specify ascp location by calling:
+    # The user can specify ascp location by calling:
     # Installation.instance.use_ascp_from_product(product_name)
     # or
     # Installation.instance.ascp_path=""
@@ -28,7 +34,7 @@ module Aspera
       include Singleton
       # protobuf generated files from sdk
       EXT_RUBY_PROTOBUF = '_pb.rb'
-      RB_SDK_FOLDER = 'lib'
+      RB_SDK_SUBFOLDER = 'lib'
       DEFAULT_ASPERA_CONF = <<~END_OF_CONFIG_FILE
         <?xml version='1.0' encoding='UTF-8'?>
         <CONF version="2">
@@ -43,11 +49,38 @@ module Aspera
       # all ascp files (in SDK)
       EXE_FILES = %i[ascp ascp4 async].freeze
       FILES = %i[transferd ssh_private_dsa ssh_private_rsa aspera_license aspera_conf fallback_certificate fallback_private_key].unshift(*EXE_FILES).freeze
-      private_constant :EXT_RUBY_PROTOBUF, :RB_SDK_FOLDER, :DEFAULT_ASPERA_CONF, :FILES
+      TRANSFER_SDK_LOCATION_URL = 'https://ibm.biz/sdk_location'
+      FILE_SCHEME_PREFIX = 'file:///'
+      SDK_ARCHIVE_FOLDERS = ['/bin/', '/aspera/'].freeze
+      # filename for ascp with optional extension (Windows)
+      private_constant :EXT_RUBY_PROTOBUF, :RB_SDK_SUBFOLDER, :DEFAULT_ASPERA_CONF, :FILES, :TRANSFER_SDK_LOCATION_URL, :FILE_SCHEME_PREFIX
       # options for SSH client private key
       CLIENT_SSH_KEY_OPTIONS = %i{dsa_rsa rsa per_client}.freeze
+
+      class << self
+        def transfer_sdk_location_url
+          ENV.fetch('ASCLI_TRANSFER_SDK_LOCATION_URL', TRANSFER_SDK_LOCATION_URL)
+        end
+
+        # Loads YAML from cloud with locations of SDK archives for all platforms
+        # @return location structure
+        def sdk_locations
+          location_url = transfer_sdk_location_url
+          transferd_locations = UriReader.read(location_url)
+          Log.log.debug{"Retrieving SDK locations from #{location_url}"}
+          begin
+            return YAML.load(transferd_locations)
+          rescue Psych::SyntaxError
+            raise "Error when parsing yaml data from: #{location_url}"
+          end
+        end
+      end
+
       # set ascp executable path
       def ascp_path=(v)
+        Aspera.assert_type(v, String)
+        Aspera.assert(!v.empty?) {'ascp path cannot be empty: check your config file'}
+        Aspera.assert(File.exist?(v)) {"No such file: [#{v}]"}
         @path_to_ascp = v
       end
 
@@ -55,37 +88,19 @@ module Aspera
         path(:ascp)
       end
 
-      def sdk_ruby_folder
-        ruby_pb_folder = File.join(sdk_folder, RB_SDK_FOLDER)
-        FileUtils.mkdir_p(ruby_pb_folder)
-        return ruby_pb_folder
-      end
-
-      # location of SDK files
+      # Compatibility
       def sdk_folder=(v)
-        Log.log.debug{"sdk_folder=#{v}"}
-        @sdk_dir = v
-        sdk_folder
-      end
-
-      # backward compatibility in sample program
-      alias_method :folder=, :sdk_folder=
-
-      # @return the path to folder where SDK is installed
-      def sdk_folder
-        raise 'SDK path was ot initialized' if @sdk_dir.nil?
-        FileUtils.mkdir_p(@sdk_dir)
-        @sdk_dir
+        Products::Transferd.sdk_directory = v
       end
 
       # find ascp in named product (use value : FIRST_FOUND='FIRST' to just use first one)
-      # or select one from Products.installed_products()
+      # or select one from installed_products()
       def use_ascp_from_product(product_name)
         if product_name.eql?(FIRST_FOUND)
-          pl = Products.installed_products.first
-          raise "no FASP installation found\nPlease check manual on how to install FASP." if pl.nil?
+          pl = installed_products.first
+          raise "no Aspera transfer module or SDK found.\nRefer to the manual or install SDK with command:\nascli conf ascp install" if pl.nil?
         else
-          pl = Products.installed_products.find{|i|i[:name].eql?(product_name)}
+          pl = installed_products.find{|i|i[:name].eql?(product_name)}
           raise "no such product installed: #{product_name}" if pl.nil?
         end
         self.ascp_path = pl[:ascp_path]
@@ -98,6 +113,8 @@ module Aspera
           m[v.to_s] =
             begin
               path(v)
+            rescue Errno::ENOENT => e
+              e.message.gsub(/.*assertion failed: /, '').gsub(/\): .*/, ')')
             rescue => e
               e.message
             end
@@ -105,7 +122,7 @@ module Aspera
       end
 
       def check_or_create_sdk_file(filename, force: false, &block)
-        return Environment.write_file_restricted(File.join(sdk_folder, filename), force: force, mode: 0o644, &block)
+        return Environment.write_file_restricted(File.join(Products::Transferd.sdk_directory, filename), force: force, mode: 0o644, &block)
       end
 
       # get path of one resource file of currently activated product
@@ -120,7 +137,7 @@ module Aspera
           file = @path_to_ascp.gsub('ascp', k.to_s)
         when :transferd
           file_is_optional = true
-          file = transferd_filepath
+          file = Products::Transferd.transferd_path
         when :ssh_private_dsa, :ssh_private_rsa
           # assume last 3 letters are type
           type = k.to_s[-3..-1].to_sym
@@ -130,8 +147,8 @@ module Aspera
         when :aspera_conf
           file = check_or_create_sdk_file('aspera.conf') {DEFAULT_ASPERA_CONF}
         when :fallback_certificate, :fallback_private_key
-          file_key = File.join(sdk_folder, 'aspera_fallback_cert_private_key.pem')
-          file_cert = File.join(sdk_folder, 'aspera_fallback_cert.pem')
+          file_key = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert_private_key.pem')
+          file_cert = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert.pem')
           if !File.exist?(file_key) || !File.exist?(file_cert)
             require 'openssl'
             # create new self signed certificate for http fallback
@@ -145,7 +162,7 @@ module Aspera
         else Aspera.error_unexpected_value(k)
         end
         return nil if file_is_optional && !File.exist?(file)
-        Aspera.assert(File.exist?(file)){"no such file: #{file}"}
+        Aspera.assert(File.exist?(file), exception_class: Errno::ENOENT){"#{k} not found (#{file})"}
         return file
       end
 
@@ -177,7 +194,7 @@ module Aspera
         return nil unless File.exist?(exe_path)
         exe_version = nil
         cmd_out = %x("#{exe_path}" #{vers_arg})
-        raise "An error occurred when testing #{ascp_filename}: #{cmd_out}" unless $CHILD_STATUS == 0
+        raise "An error occurred when testing #{exe_path}: #{cmd_out}" unless $CHILD_STATUS == 0
         # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
         m = cmd_out.match(/ version ([0-9.]+)/)
         exe_version = m[1].gsub(/\.$/, '') unless m.nil?
@@ -229,67 +246,127 @@ module Aspera
         return data
       end
 
+      # information for `ascp info`
       def ascp_info
-        files = file_paths
-        return files.merge(ascp_pvcl_info).merge(ascp_ssl_info)
+        ascp_data = file_paths
+        ascp_data.merge!(ascp_pvcl_info)
+        ascp_data['sdk_locations'] = self.class.transfer_sdk_location_url
+        ascp_data.merge!(ascp_ssl_info)
+        return ascp_data
+      end
+
+      # @return the url for download of SDK archive for the given platform and version
+      def sdk_url_for_platform(platform: nil, version: nil)
+        locations = self.class.sdk_locations
+        platform = Environment.architecture if platform.nil?
+        locations = locations.select{|l|l['platform'].eql?(platform)}
+        raise "No SDK for platform: #{platform}" if locations.empty?
+        version = locations.max_by { |entry| Gem::Version.new(entry['version']) }['version'] if version.nil?
+        info = locations.select{|entry| entry['version'].eql?(version)}
+        raise "No such version: #{version} for #{platform}" if info.empty?
+        return info.first['url']
+      end
+
+      # @param &block called with entry information
+      def extract_archive_files(sdk_archive_path)
+        raise 'missing block' unless block_given?
+        case sdk_archive_path
+        # Windows and Mac use zip
+        when /\.zip$/
+          require 'zip'
+          # extract files from archive
+          Zip::File.open(sdk_archive_path) do |zip_file|
+            zip_file.each do |entry|
+              next if entry.name.end_with?('/')
+              yield(entry.name, entry.get_input_stream, nil)
+            end
+          end
+        # Other Unixes use tar.gz
+        when /\.tar\.gz/
+          require 'zlib'
+          require 'rubygems/package'
+          Zlib::GzipReader.open(sdk_archive_path) do |gzip|
+            Gem::Package::TarReader.new(gzip) do |tar|
+              tar.each do |entry|
+                next if entry.directory?
+                yield(entry.full_name, entry, entry.symlink? ? entry.header.linkname : nil)
+              end
+            end
+          end
+        else
+          raise "unknown archive extension: #{sdk_archive_path}"
+        end
       end
 
       # download aspera SDK or use local file
       # extracts ascp binary for current system architecture
+      # @param url      [String] URL to SDK archive, or SpecialValues::DEF
+      # @param folder   [String] destination
+      # @param backup   [Bool]
+      # @param with_exe [Bool]
+      # @param &block   [Proc] a lambda that receives a file path from archive and tells detination sub folder, or nil to not extract
       # @return ascp version (from execution)
-      def install_sdk(sdk_url)
-        # SDK is organized by architecture, check this first, in case architecture is not supported
-        arch_filter = "#{Environment.architecture}/"
-        require 'zip'
-        sdk_zip_path = File.join(Dir.tmpdir, 'sdk.zip')
-        if sdk_url.start_with?('file:')
+      def install_sdk(url: nil, version: nil, folder: nil, backup: true, with_exe: true, &block)
+        url = sdk_url_for_platform(version: version) if url.nil? || url.eql?('DEF')
+        folder = Products::Transferd.sdk_directory if folder.nil?
+        subfolder_lambda = block
+        if subfolder_lambda.nil?
+          subfolder_lambda = ->(name) do
+            if SDK_ARCHIVE_FOLDERS.any?{|i|name.include?(i)}
+              '/'
+            elsif name.end_with?(EXT_RUBY_PROTOBUF)
+              RB_SDK_SUBFOLDER
+            end
+          end
+        end
+        if url.start_with?('file:')
           # require specific file scheme: the path part is "relative", or absolute if there are 4 slash
-          raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
-          sdk_zip_path = sdk_url.gsub(%r{^file:///}, '')
+          raise 'use format: file:///<path>' unless url.start_with?(FILE_SCHEME_PREFIX)
+          sdk_archive_path = url[FILE_SCHEME_PREFIX.length..-1]
+          delete_archive = false
         else
-          Aspera::Rest.new(base_url: sdk_url, redirect_max: 3).call(operation: 'GET', save_to_file: sdk_zip_path)
+          sdk_archive_path = File.join(Dir.tmpdir, File.basename(url))
+          Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to_file: sdk_archive_path)
+          delete_archive = true
         end
         # rename old install
-        if !Dir.empty?(sdk_folder)
+        if backup && !Dir.empty?(folder)
           Log.log.warn('Previous install exists, renaming folder.')
-          File.rename(sdk_folder, "#{sdk_folder}.#{Time.now.strftime('%Y%m%d%H%M%S')}")
+          File.rename(folder, "#{folder}.#{Time.now.strftime('%Y%m%d%H%M%S')}")
           # TODO: delete old archives ?
         end
-        # extract files from archive
-        Zip::File.open(sdk_zip_path) do |zip_file|
-          zip_file.each do |entry|
-            # skip folder entries
-            next if entry.name.end_with?('/')
-            dest_folder = nil
-            # binaries
-            dest_folder = sdk_folder if entry.name.include?(arch_filter)
-            # ruby adapters
-            dest_folder = sdk_ruby_folder if entry.name.end_with?(EXT_RUBY_PROTOBUF)
-            next if dest_folder.nil?
-            File.open(File.join(dest_folder, File.basename(entry.name)), 'wb') do |output_stream|
-              IO.copy_stream(entry.get_input_stream, output_stream)
-            end
+        extract_archive_files(sdk_archive_path) do |entry_name, entry_stream, link_target|
+          subfolder = subfolder_lambda.call(entry_name)
+          next if subfolder.nil?
+          dest_folder = File.join(folder, subfolder)
+          FileUtils.mkdir_p(dest_folder)
+          new_file = File.join(dest_folder, File.basename(entry_name))
+          if link_target.nil?
+            File.open(new_file, 'wb') { |output_stream|IO.copy_stream(entry_stream, output_stream)}
+          else
+            File.symlink(link_target, new_file)
           end
         end
-        File.unlink(sdk_zip_path) rescue nil # Windows may give error
+        File.unlink(sdk_archive_path) rescue nil if delete_archive # Windows may give error
+        return unless with_exe
         # ensure license file are generated so that ascp invocation for version works
         path(:aspera_license)
         path(:aspera_conf)
-        sdk_ascp_file = Products.ascp_filename
-        sdk_ascp_path = File.join(sdk_folder, sdk_ascp_file)
+        sdk_ascp_file = Environment.exe_file('ascp')
+        sdk_ascp_path = File.join(folder, sdk_ascp_file)
         raise "No #{sdk_ascp_file} found in SDK archive" unless File.exist?(sdk_ascp_path)
         EXE_FILES.each do |exe_sym|
           exe_path = sdk_ascp_path.gsub('ascp', exe_sym.to_s)
           Environment.restrict_file_access(exe_path, mode: 0o755) if File.exist?(exe_path)
         end
         sdk_ascp_version = get_ascp_version(sdk_ascp_path)
-        sdk_daemon_path = transferd_filepath
+        sdk_daemon_path = Products::Transferd.transferd_path
         Log.log.warn{"No #{sdk_daemon_path} in SDK archive"} unless File.exist?(sdk_daemon_path)
         Environment.restrict_file_access(sdk_daemon_path, mode: 0o755) if File.exist?(sdk_daemon_path)
         transferd_version = get_exe_version(sdk_daemon_path, 'version')
         sdk_name = 'IBM Aspera Transfer SDK'
         sdk_version = transferd_version || sdk_ascp_version
-        File.write(File.join(sdk_folder, Products::INFO_META_FILE), "<product><name>#{sdk_name}</name><version>#{sdk_version}</version></product>")
+        File.write(File.join(folder, Products::Other::INFO_META_FILE), "<product><name>#{sdk_name}</name><version>#{sdk_version}</version></product>")
         return sdk_name, sdk_version
       end
 
@@ -301,10 +378,29 @@ module Aspera
       def initialize
         @path_to_ascp = nil
         @sdk_dir = nil
+        @found_products = nil
       end
 
-      def transferd_filepath
-        return File.join(sdk_folder, 'asperatransferd' + Environment.exe_extension) # cspell:disable-line
+      public
+
+      # @return the list of installed products in format of product_locations_on_current_os
+      def installed_products
+        if @found_products.nil?
+          # :expected  M app name is taken from the manifest if present, else defaults to this value
+          # :app_root  M main folder for the application
+          # :log_root  O location of log files (Linux uses syslog)
+          # :run_root  O only for Connect Client, location of http port file
+          # :sub_bin   O subfolder with executables, default : bin
+          scan_locations = Products::Transferd.locations.concat(
+            Products::Alpha.locations,
+            Products::Connect.locations,
+            Products::Other::LOCATION_ON_THIS_OS
+          )
+          # .each {|item| item.deep_do {|h, _k, _v, _m|h.freeze}}.freeze
+          # search installed products: with ascp
+          @found_products = Products::Other.find(scan_locations)
+        end
+        return @found_products
       end
     end
   end

data/lib/aspera/ascp/management.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'aspera/assert'
+
 module Aspera
   module Ascp
     # processing of ascp management port events
@@ -196,18 +198,39 @@ module Aspera
       BOOLEAN_FIELDS = %w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled
                           MoveRange Keepalive TestLogin UseProxy Precalc RTTAutocorrect].freeze
       BOOLEAN_TRUE = 'Yes'
+
+      private_constant :OPERATIONS, :PARAMETERS, :MGT_HEADER, :MGT_FRAME_SEPARATOR, :INTEGER_FIELDS, :BOOLEAN_FIELDS, :BOOLEAN_TRUE
       # cspell: enable
 
       class << self
         # translates mgt port event into (enhanced) typed event
         def enhanced_event_format(event)
           return event.keys.each_with_object({}) do |e, h|
-            new_name = e.capital_to_snake.gsub(/(usec)$/, '_\1').downcase
-            value = event[e]
-            value = value.to_i if INTEGER_FIELDS.include?(e)
-            value = value.eql?(BOOLEAN_TRUE) if BOOLEAN_FIELDS.include?(e)
-            h[new_name] = value
-          end
+                   new_name =
+                     case e
+                     when 'Elapsedusec' then 'elapsed_usec'
+                     when 'Bytescont' then 'bytes_cont'
+                     else e.capital_to_snake
+                     end
+                   h[new_name] =
+                     if INTEGER_FIELDS.include?(e) then event[e].to_i
+                     elsif BOOLEAN_FIELDS.include?(e) then event[e].eql?(BOOLEAN_TRUE)
+                     else
+                       event[e]
+                     end
+                 end
+        end
+
+        # build command to send on management port
+        # @param data [Hash] {'type'=>'START','source'=>_path_,'destination'=>_path_}
+        def command_to_stream(data)
+          # TODO: translate enhanced to capitalized ?
+          data
+            .keys
+            .map{|k|"#{k.capitalize}: #{data[k]}"}
+            .unshift(MGT_HEADER)
+            .push('', '')
+            .join("\n")
         end
       end
 
@@ -237,8 +260,7 @@ module Aspera
           @last_event = @event_build
           @event_build = nil
           return @last_event
-        else
-          raise "mgt port: unexpected line: [#{line}]"
+        else Aspera.error_unexpected_value(line){'mgt port'}
         end
         return nil
       end

data/lib/aspera/assert.rb

@@ -8,13 +8,13 @@ module Aspera
   end
   class << self
     # the block is executed in the context of the Aspera module
-    def assert(assertion, info = nil, level: 2, exception_class: AssertError)
+    def assert(assertion, info = nil, exception_class: AssertError)
       raise InternalError, 'bad assert: both info and block given' unless info.nil? || !block_given?
       return if assertion
       message = 'assertion failed'
       info = yield if block_given?
       message = "#{message}: #{info}" if info
-      message = "#{message}: #{caller(level..level).first}"
+      message = "#{message}: #{caller.find{|call|!call.start_with?(__FILE__)}}"
       raise exception_class, message
     end
 
@@ -22,13 +22,18 @@ module Aspera
     # @param value [Object] the value to check
     # @param type [Class] the expected type
     def assert_type(value, type, exception_class: AssertError)
-      assert(value.is_a?(type), level: 3, exception_class: exception_class){"#{block_given? ? "#{yield}: " : nil}expecting #{type}, but have #{value.inspect}"}
+      assert(value.is_a?(type), exception_class: exception_class){"#{block_given? ? "#{yield}: " : nil}expecting #{type}, but have #{value.inspect}"}
     end
 
     # assert that value is one of the given values
+    # @param value value to check
+    # @param values accepted values
+    # @param exception_class exception in case of no match
     def assert_values(value, values, exception_class: AssertError)
-      assert(values.include?(value), level: 3, exception_class: exception_class) do
-        "#{block_given? ? "#{yield}: " : nil}expecting one of #{values.inspect}, but have #{value.inspect}"
+      assert(values.include?(value), exception_class: exception_class) do
+        val_list = values.inspect
+        val_list = "one of #{val_list}" if values.is_a?(Array)
+        "#{block_given? ? "#{yield}: " : nil}expecting #{val_list}, but have #{value.inspect}"
       end
     end