aspera-cli 4.20.0 → 4.21.1

data/lib/aspera/ascp/installation.rb

@@ -3,11 +3,17 @@
 # cspell:ignore protobuf ckpt
 require 'aspera/environment'
 require 'aspera/data_repository'
-require 'aspera/ascp/products'
 require 'aspera/log'
 require 'aspera/rest'
+require 'aspera/uri_reader'
 require 'aspera/assert'
 require 'aspera/web_server_simple'
+require 'aspera/cli/info'
+require 'aspera/cli/version'
+require 'aspera/products/alpha'
+require 'aspera/products/connect'
+require 'aspera/products/transferd'
+require 'aspera/products/other'
 require 'English'
 require 'singleton'
 require 'xmlsimple'
@@ -20,7 +26,7 @@ module Aspera
     # Singleton that tells where to find ascp and other local resources (keys..) , using the "path(:name)" method.
     # It is used by object : AgentDirect to find necessary resources
     # By default it takes the first Aspera product found
-    # but the user can specify ascp location by calling:
+    # The user can specify ascp location by calling:
     # Installation.instance.use_ascp_from_product(product_name)
     # or
     # Installation.instance.ascp_path=""
@@ -46,12 +52,35 @@ module Aspera
       TRANSFER_SDK_LOCATION_URL = 'https://ibm.biz/sdk_location'
       FILE_SCHEME_PREFIX = 'file:///'
       SDK_ARCHIVE_FOLDERS = ['/bin/', '/aspera/'].freeze
+      # filename for ascp with optional extension (Windows)
       private_constant :EXT_RUBY_PROTOBUF, :RB_SDK_SUBFOLDER, :DEFAULT_ASPERA_CONF, :FILES, :TRANSFER_SDK_LOCATION_URL, :FILE_SCHEME_PREFIX
       # options for SSH client private key
       CLIENT_SSH_KEY_OPTIONS = %i{dsa_rsa rsa per_client}.freeze
 
+      class << self
+        def transfer_sdk_location_url
+          ENV.fetch('ASCLI_TRANSFER_SDK_LOCATION_URL', TRANSFER_SDK_LOCATION_URL)
+        end
+
+        # Loads YAML from cloud with locations of SDK archives for all platforms
+        # @return location structure
+        def sdk_locations
+          location_url = transfer_sdk_location_url
+          transferd_locations = UriReader.read(location_url)
+          Log.log.debug{"Retrieving SDK locations from #{location_url}"}
+          begin
+            return YAML.load(transferd_locations)
+          rescue Psych::SyntaxError
+            raise "Error when parsing yaml data from: #{location_url}"
+          end
+        end
+      end
+
       # set ascp executable path
       def ascp_path=(v)
+        Aspera.assert_type(v, String)
+        Aspera.assert(!v.empty?) {'ascp path cannot be empty: check your config file'}
+        Aspera.assert(File.exist?(v)) {"No such file: [#{v}]"}
         @path_to_ascp = v
       end
 
@@ -59,31 +88,19 @@ module Aspera
         path(:ascp)
       end
 
-      # location of SDK files
+      # Compatibility
       def sdk_folder=(v)
-        Log.log.debug{"sdk_folder=#{v}"}
-        @sdk_dir = v
-        sdk_folder
-      end
-
-      # backward compatibility in sample program
-      alias_method :folder=, :sdk_folder=
-
-      # @return the path to folder where SDK is installed
-      def sdk_folder
-        raise 'SDK path was ot initialized' if @sdk_dir.nil?
-        FileUtils.mkdir_p(@sdk_dir)
-        @sdk_dir
+        Products::Transferd.sdk_directory = v
       end
 
       # find ascp in named product (use value : FIRST_FOUND='FIRST' to just use first one)
-      # or select one from Products.installed_products()
+      # or select one from installed_products()
       def use_ascp_from_product(product_name)
         if product_name.eql?(FIRST_FOUND)
-          pl = Products.installed_products.first
-          raise "ascp found: no Aspera transfer module or SDK found.\nRefer to the manual or install SDK with command:\nascli conf ascp install" if pl.nil?
+          pl = installed_products.first
+          raise "no Aspera transfer module or SDK found.\nRefer to the manual or install SDK with command:\nascli conf ascp install" if pl.nil?
         else
-          pl = Products.installed_products.find{|i|i[:name].eql?(product_name)}
+          pl = installed_products.find{|i|i[:name].eql?(product_name)}
           raise "no such product installed: #{product_name}" if pl.nil?
         end
         self.ascp_path = pl[:ascp_path]
@@ -96,6 +113,8 @@ module Aspera
           m[v.to_s] =
             begin
               path(v)
+            rescue Errno::ENOENT => e
+              e.message.gsub(/.*assertion failed: /, '').gsub(/\): .*/, ')')
             rescue => e
               e.message
             end
@@ -103,7 +122,7 @@ module Aspera
       end
 
       def check_or_create_sdk_file(filename, force: false, &block)
-        return Environment.write_file_restricted(File.join(sdk_folder, filename), force: force, mode: 0o644, &block)
+        return Environment.write_file_restricted(File.join(Products::Transferd.sdk_directory, filename), force: force, mode: 0o644, &block)
       end
 
       # get path of one resource file of currently activated product
@@ -118,7 +137,7 @@ module Aspera
           file = @path_to_ascp.gsub('ascp', k.to_s)
         when :transferd
           file_is_optional = true
-          file = transferd_filepath
+          file = Products::Transferd.transferd_path
         when :ssh_private_dsa, :ssh_private_rsa
           # assume last 3 letters are type
           type = k.to_s[-3..-1].to_sym
@@ -128,8 +147,8 @@ module Aspera
         when :aspera_conf
           file = check_or_create_sdk_file('aspera.conf') {DEFAULT_ASPERA_CONF}
         when :fallback_certificate, :fallback_private_key
-          file_key = File.join(sdk_folder, 'aspera_fallback_cert_private_key.pem')
-          file_cert = File.join(sdk_folder, 'aspera_fallback_cert.pem')
+          file_key = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert_private_key.pem')
+          file_cert = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert.pem')
           if !File.exist?(file_key) || !File.exist?(file_cert)
             require 'openssl'
             # create new self signed certificate for http fallback
@@ -143,7 +162,7 @@ module Aspera
         else Aspera.error_unexpected_value(k)
         end
         return nil if file_is_optional && !File.exist?(file)
-        Aspera.assert(File.exist?(file)){"no such file: #{file}"}
+        Aspera.assert(File.exist?(file), exception_class: Errno::ENOENT){"#{k} not found (#{file})"}
         return file
       end
 
@@ -231,21 +250,14 @@ module Aspera
       def ascp_info
         ascp_data = file_paths
         ascp_data.merge!(ascp_pvcl_info)
-        ascp_data['sdk_locations'] = TRANSFER_SDK_LOCATION_URL
+        ascp_data['sdk_locations'] = self.class.transfer_sdk_location_url
         ascp_data.merge!(ascp_ssl_info)
         return ascp_data
       end
 
-      # Loads YAML from cloud with locations of SDK archives for all platforms
-      # @return location structure
-      def sdk_locations
-        yaml_text = Aspera::Rest.new(base_url: TRANSFER_SDK_LOCATION_URL, redirect_max: 3).call(operation: 'GET')[:data]
-        YAML.load(yaml_text)
-      end
-
       # @return the url for download of SDK archive for the given platform and version
       def sdk_url_for_platform(platform: nil, version: nil)
-        locations = sdk_locations
+        locations = self.class.sdk_locations
         platform = Environment.architecture if platform.nil?
         locations = locations.select{|l|l['platform'].eql?(platform)}
         raise "No SDK for platform: #{platform}" if locations.empty?
@@ -255,6 +267,7 @@ module Aspera
         return info.first['url']
       end
 
+      # @param &block called with entry information
       def extract_archive_files(sdk_archive_path)
         raise 'missing block' unless block_given?
         case sdk_archive_path
@@ -265,7 +278,7 @@ module Aspera
           Zip::File.open(sdk_archive_path) do |zip_file|
             zip_file.each do |entry|
               next if entry.name.end_with?('/')
-              yield(entry.name, entry.get_input_stream)
+              yield(entry.name, entry.get_input_stream, nil)
             end
           end
         # Other Unixes use tar.gz
@@ -276,7 +289,7 @@ module Aspera
             Gem::Package::TarReader.new(gzip) do |tar|
               tar.each do |entry|
                 next if entry.directory?
-                yield(entry.full_name, entry)
+                yield(entry.full_name, entry, entry.symlink? ? entry.header.linkname : nil)
               end
             end
           end
@@ -287,11 +300,15 @@ module Aspera
 
       # download aspera SDK or use local file
       # extracts ascp binary for current system architecture
-      # @param url [String] URL to SDK archive, or SpecialValues::DEF
+      # @param url      [String] URL to SDK archive, or SpecialValues::DEF
+      # @param folder   [String] destination
+      # @param backup   [Bool]
+      # @param with_exe [Bool]
+      # @param &block   [Proc] a lambda that receives a file path from archive and tells detination sub folder, or nil to not extract
       # @return ascp version (from execution)
-      def install_sdk(url: nil, folder: nil, backup: true, with_exe: true, &block)
-        url = sdk_url_for_platform if url.nil? || url.eql?('DEF')
-        folder = sdk_folder if folder.nil?
+      def install_sdk(url: nil, version: nil, folder: nil, backup: true, with_exe: true, &block)
+        url = sdk_url_for_platform(version: version) if url.nil? || url.eql?('DEF')
+        folder = Products::Transferd.sdk_directory if folder.nil?
         subfolder_lambda = block
         if subfolder_lambda.nil?
           subfolder_lambda = ->(name) do
@@ -318,13 +335,16 @@ module Aspera
           File.rename(folder, "#{folder}.#{Time.now.strftime('%Y%m%d%H%M%S')}")
           # TODO: delete old archives ?
         end
-        extract_archive_files(sdk_archive_path) do |entry_name, entry_stream|
+        extract_archive_files(sdk_archive_path) do |entry_name, entry_stream, link_target|
           subfolder = subfolder_lambda.call(entry_name)
           next if subfolder.nil?
           dest_folder = File.join(folder, subfolder)
           FileUtils.mkdir_p(dest_folder)
-          File.open(File.join(dest_folder, File.basename(entry_name)), 'wb') do |output_stream|
-            IO.copy_stream(entry_stream, output_stream)
+          new_file = File.join(dest_folder, File.basename(entry_name))
+          if link_target.nil?
+            File.open(new_file, 'wb') { |output_stream|IO.copy_stream(entry_stream, output_stream)}
+          else
+            File.symlink(link_target, new_file)
           end
         end
         File.unlink(sdk_archive_path) rescue nil if delete_archive # Windows may give error
@@ -332,7 +352,7 @@ module Aspera
         # ensure license file are generated so that ascp invocation for version works
         path(:aspera_license)
         path(:aspera_conf)
-        sdk_ascp_file = Products.ascp_filename
+        sdk_ascp_file = Environment.exe_file('ascp')
         sdk_ascp_path = File.join(folder, sdk_ascp_file)
         raise "No #{sdk_ascp_file} found in SDK archive" unless File.exist?(sdk_ascp_path)
         EXE_FILES.each do |exe_sym|
@@ -340,13 +360,13 @@ module Aspera
           Environment.restrict_file_access(exe_path, mode: 0o755) if File.exist?(exe_path)
         end
         sdk_ascp_version = get_ascp_version(sdk_ascp_path)
-        sdk_daemon_path = transferd_filepath
+        sdk_daemon_path = Products::Transferd.transferd_path
         Log.log.warn{"No #{sdk_daemon_path} in SDK archive"} unless File.exist?(sdk_daemon_path)
         Environment.restrict_file_access(sdk_daemon_path, mode: 0o755) if File.exist?(sdk_daemon_path)
         transferd_version = get_exe_version(sdk_daemon_path, 'version')
         sdk_name = 'IBM Aspera Transfer SDK'
         sdk_version = transferd_version || sdk_ascp_version
-        File.write(File.join(folder, Products::INFO_META_FILE), "<product><name>#{sdk_name}</name><version>#{sdk_version}</version></product>")
+        File.write(File.join(folder, Products::Other::INFO_META_FILE), "<product><name>#{sdk_name}</name><version>#{sdk_version}</version></product>")
         return sdk_name, sdk_version
       end
 
@@ -358,10 +378,29 @@ module Aspera
       def initialize
         @path_to_ascp = nil
         @sdk_dir = nil
+        @found_products = nil
       end
 
-      def transferd_filepath
-        return File.join(sdk_folder, 'asperatransferd' + Environment.exe_extension) # cspell:disable-line
+      public
+
+      # @return the list of installed products in format of product_locations_on_current_os
+      def installed_products
+        if @found_products.nil?
+          # :expected  M app name is taken from the manifest if present, else defaults to this value
+          # :app_root  M main folder for the application
+          # :log_root  O location of log files (Linux uses syslog)
+          # :run_root  O only for Connect Client, location of http port file
+          # :sub_bin   O subfolder with executables, default : bin
+          scan_locations = Products::Transferd.locations.concat(
+            Products::Alpha.locations,
+            Products::Connect.locations,
+            Products::Other::LOCATION_ON_THIS_OS
+          )
+          # .each {|item| item.deep_do {|h, _k, _v, _m|h.freeze}}.freeze
+          # search installed products: with ascp
+          @found_products = Products::Other.find(scan_locations)
+        end
+        return @found_products
       end
     end
   end

data/lib/aspera/ascp/management.rb

@@ -198,18 +198,39 @@ module Aspera
       BOOLEAN_FIELDS = %w[Encryption Remote RateLock MinRateLock PolicyLock FilesEncrypt FilesDecrypt VLinkLocalEnabled VLinkRemoteEnabled
                           MoveRange Keepalive TestLogin UseProxy Precalc RTTAutocorrect].freeze
       BOOLEAN_TRUE = 'Yes'
+
+      private_constant :OPERATIONS, :PARAMETERS, :MGT_HEADER, :MGT_FRAME_SEPARATOR, :INTEGER_FIELDS, :BOOLEAN_FIELDS, :BOOLEAN_TRUE
       # cspell: enable
 
       class << self
         # translates mgt port event into (enhanced) typed event
         def enhanced_event_format(event)
           return event.keys.each_with_object({}) do |e, h|
-            new_name = e.capital_to_snake.gsub(/(usec)$/, '_\1').downcase
-            value = event[e]
-            value = value.to_i if INTEGER_FIELDS.include?(e)
-            value = value.eql?(BOOLEAN_TRUE) if BOOLEAN_FIELDS.include?(e)
-            h[new_name] = value
-          end
+                   new_name =
+                     case e
+                     when 'Elapsedusec' then 'elapsed_usec'
+                     when 'Bytescont' then 'bytes_cont'
+                     else e.capital_to_snake
+                     end
+                   h[new_name] =
+                     if INTEGER_FIELDS.include?(e) then event[e].to_i
+                     elsif BOOLEAN_FIELDS.include?(e) then event[e].eql?(BOOLEAN_TRUE)
+                     else
+                       event[e]
+                     end
+                 end
+        end
+
+        # build command to send on management port
+        # @param data [Hash] {'type'=>'START','source'=>_path_,'destination'=>_path_}
+        def command_to_stream(data)
+          # TODO: translate enhanced to capitalized ?
+          data
+            .keys
+            .map{|k|"#{k.capitalize}: #{data[k]}"}
+            .unshift(MGT_HEADER)
+            .push('', '')
+            .join("\n")
         end
       end