aspera-cli 4.23.0 → 4.24.1

data/lib/aspera/sync/database.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+if defined?(JRUBY_VERSION)
+  require 'jdbc/sqlite3'
+  Jdbc::SQLite3.load_driver
+  require 'sequel'
+else
+  require 'sqlite3'
+end
+
+# A wrapper class that provides common API for Ruby and JRuby
+class SqLite3Wrapper
+  def initialize(db_path)
+    @db_path = db_path
+  end
+
+  if defined?(JRUBY_VERSION)
+    def execute(sql)
+      db = Sequel.connect("jdbc:sqlite:#{@db_path}")
+      begin
+        normalize_rows(db.fetch(sql).all)
+      ensure
+        db.disconnect
+      end
+    end
+  else
+    def execute(sql)
+      db = SQLite3::Database.new(@db_path).tap{ |d| d.results_as_hash = true}
+      begin
+        normalize_rows(db.execute(sql))
+      ensure
+        db.close
+      end
+    end
+  end
+
+  # The table contains a single row
+  def single_table(table_name)
+    execute("SELECT * FROM #{table_name} LIMIT 1").first
+  end
+
+  def full_table(table_name)
+    execute("SELECT * FROM #{table_name}")
+  end
+
+  private
+
+  def normalize_rows(rows)
+    rows.map{ |r| r.transform_keys(&:to_s)}
+  end
+end
+
+module Aspera
+  module Sync
+    class Database
+      def initialize(db_path)
+        @db = SqLite3Wrapper.new(db_path)
+      end
+
+      def overview
+        tables = @db.execute("SELECT name FROM sqlite_master WHERE type='table';")
+        tables.flat_map do |table_row|
+          table_name = table_row['name']
+          @db.execute("PRAGMA table_info(#{table_name});").map do |col|
+            {'table' => table_name}.merge(col)
+          end
+        end
+      end
+
+      def meta
+        @db.single_table('sync_snapmeta_table')
+      end
+
+      def counters
+        @db.single_table('sync_snap_counters_table')
+      end
+
+      def file_info
+        @db.full_table('sync_snapdb_table')
+      end
+    end
+  end
+end

data/lib/aspera/sync/operations.rb

@@ -0,0 +1,296 @@
+# frozen_string_literal: true
+
+# cspell:words logdir bidi watchd cooloff asyncadmin
+
+require 'aspera/ascp/installation'
+require 'aspera/agent/direct'
+require 'aspera/command_line_converter'
+require 'aspera/command_line_builder'
+require 'aspera/log'
+require 'aspera/assert'
+require 'aspera/environment'
+require 'json'
+require 'base64'
+require 'open3'
+require 'English'
+
+module Aspera
+  module Sync
+    # builds command line arg for async and execute it
+    module Operations
+      # sync direction
+      DIRECTIONS = %i[push pull bidi].freeze
+      # default direction for sync
+      DEFAULT_DIRECTION = :push
+
+      class << self
+        # Set `remote_dir` in sync parameters based on transfer spec
+        # @param params [Hash] Sync parameters, old or new format
+        # @param remote_dir_key [String] key to update in above hash
+        # @param transfer_spec [Hash] transfer spec
+        def update_remote_dir(params, remote_dir_key, transfer_spec)
+          if transfer_spec.dig(*%w[tags aspera node file_id])
+            # in AoC, use gen4
+            params[remote_dir_key] = '/'
+          elsif transfer_spec['cookie']&.start_with?('aspera.shares2')
+            # TODO : something more generic, independent of Shares
+            # in Shares, the actual folder on remote end is not always the same as the name of the share
+            remote_key = transfer_spec['direction'].eql?('send') ? 'destination' : 'source'
+            actual_remote = transfer_spec['paths']&.first&.[](remote_key)
+            params[remote_dir_key] = actual_remote if actual_remote
+          end
+          nil
+        end
+
+        # Get certificates to use for remote connection
+        # @param remote [Hash] remote connection parameters
+        # @return [Array<String>] list of certificate file paths
+        def remote_certificates(remote)
+          certificates_to_use = []
+          # use web socket secure for session ?
+          if remote['connect_mode']&.eql?('ws')
+            remote.delete('port')
+            remote.delete('fingerprint')
+            # ignore cert for wss ?
+            # if @options[:check_ignore_cb]&.call(remote['host'], remote['ws_port'])
+            #   wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
+            #   wss_url = "https://#{remote['host']}:#{remote['ws_port']}"
+            #   File.write(wss_cert_file, Rest.remote_certificate_chain(wss_url))
+            #   certificates_to_use.push(wss_cert_file)
+            # end
+            # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
+            # certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs]
+          else
+            # remove unused parameter (avoid warning)
+            remote.delete('ws_port')
+            # add SSH bypass keys when authentication is token and no auth is provided
+            certificates_to_use.concat(Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa)) if remote.key?('token') && !remote.key?('pass')
+          end
+          return certificates_to_use
+        end
+
+        # Get symbol of sync direction, defaulting to :push
+        # @param params [Hash] Sync parameters, old or new format
+        # @return [Symbol] direction symbol, one of :push, :pull, :bidi
+        def direction_sym(params)
+          (params['direction'] || DEFAULT_DIRECTION).to_sym
+        end
+
+        # Start the sync process
+        # @param params [Hash] Sync parameters, old or new format
+        # @param opt_ts [Hash] Optional transfer spec
+        # @param &block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
+        def start(params, opt_ts = nil)
+          Log.dump(:sync_params_initial, params)
+          Aspera.assert_type(params, Hash)
+          Aspera.assert(PARAM_KEYS.any?{ |k| params.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
+          env_args = {
+            args: [],
+            env:  {}
+          }
+          if params.key?('local')
+            # "conf" format
+            Aspera.assert_type(params['local'], Hash){'local'}
+            remote = params['remote']
+            Aspera.assert_type(remote, Hash){'remote'}
+            Aspera.assert_type(remote['path'], String){'remote path'}
+            # get transfer spec if possible, and feed back to new structure
+            if block_given?
+              transfer_spec = yield(direction_sym(params), params['local']['path'], remote['path'])
+              Log.dump(:auth_ts, transfer_spec)
+              transfer_spec.deep_merge!(opt_ts) unless opt_ts.nil?
+              tspec_to_sync_info(transfer_spec, params, CONF_SCHEMA)
+              update_remote_dir(remote, 'path', transfer_spec)
+            end
+            remote['connect_mode'] ||= transfer_spec['wss_enabled'] ? 'ws' : 'ssh'
+            add_certificates = remote_certificates(remote)
+            if !add_certificates.empty?
+              remote['private_key_paths'] ||= []
+              remote['private_key_paths'].concat(add_certificates)
+            end
+            # '--exclusive-mgmt-port=12345', '--arg-err-path=-',
+            env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(params))}"]
+            Log.dump(:sync_conf, params)
+            agent = Agent::Direct.new
+            agent.start_and_monitor_process(session: {}, name: :async, **env_args)
+          else
+            # "args" format
+            raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
+              params.keys.push('instance').uniq.sort.eql?(CMDLINE_PARAMS_KEYS)
+            Aspera.assert_type(params['sessions'], Array)
+            Aspera.assert_type(params['sessions'].first, Hash)
+            if block_given?
+              params['sessions'].each do |session|
+                Aspera.assert_type(session['local_dir'], String){'local_dir'}
+                Aspera.assert_type(session['remote_dir'], String){'remote_dir'}
+                transfer_spec = yield(direction_sym(session), session['local_dir'], session['remote_dir'])
+                Log.dump(:auth_ts, transfer_spec)
+                transfer_spec.deep_merge!(opt_ts) unless opt_ts.nil?
+                tspec_to_sync_info(transfer_spec, session, SESSION_SCHEMA)
+                session['private_key_paths'] = Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa) if transfer_spec.key?('token')
+                update_remote_dir(session, 'remote_dir', transfer_spec)
+              end
+            end
+            if params.key?('instance')
+              Aspera.assert_type(params['instance'], Hash)
+              instance_builder = CommandLineBuilder.new(params['instance'], INSTANCE_SCHEMA, CommandLineConverter)
+              instance_builder.process_params
+              instance_builder.add_env_args(env_args)
+            end
+            params['sessions'].each do |session_params|
+              Aspera.assert_type(session_params, Hash)
+              Aspera.assert(session_params.key?('name')){'session must contain at least: name'}
+              session_builder = CommandLineBuilder.new(session_params, SESSION_SCHEMA, CommandLineConverter)
+              session_builder.process_params
+              session_builder.add_env_args(env_args)
+            end
+            Environment.secure_execute(exec: Ascp::Installation.instance.path(:async), **env_args)
+          end
+          return
+        end
+
+        # Parse output of asyncadmin
+        def parse_status(stdout)
+          Log.log.trace1{"stdout=#{stdout}"}
+          result = {}
+          ids = nil
+          stdout.split("\n").each do |line|
+            info = line.split(':', 2).map(&:lstrip)
+            if info[1].eql?('')
+              info[1] = ids = []
+            elsif info[1].nil?
+              ids.push(info[0])
+              next
+            end
+            result[info[0]] = info[1]
+          end
+          return result
+        end
+
+        # Run `asyncadmin` to get status of sync session
+        # @param params [Hash] sync parameters in conf or args format
+        # @return [Hash] parsed output of asyncadmin
+        def admin_status(params)
+          Aspera.assert(PARAM_KEYS.any?{ |k| params.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
+          arguments = ['--quiet']
+          if params.key?('local')
+            # "conf" format
+            arguments.push("--name=#{params['name']}")
+            if params.key?('local_db_dir')
+              arguments.push("--local-db-dir=#{params['local_db_dir']}")
+            elsif params.dig('local', 'path')
+              arguments.push("--local-dir=#{params.dig('local', 'path')}")
+            else
+              raise Error, 'Missing either local_db_dir or local.path'
+            end
+          else
+            # "args" format
+            session = params['sessions'].first
+            arguments.push("--name=#{session['name']}")
+            if session.key?('local_db_dir')
+              arguments.push("--local-db-dir=#{session['local_db_dir']}")
+            elsif session.key?('local_dir')
+              arguments.push("--local-dir=#{session['local_dir']}")
+            else
+              raise Error, 'Missing either local_db_dir or local_dir'
+            end
+          end
+          stdout = Environment.secure_capture(exec: ASYNC_ADMIN_EXECUTABLE, args: arguments)
+          return parse_status(stdout)
+        end
+
+        # Find the local database folder based on params
+        # @param params [Hash] sync parameters in conf or args format
+        # @return [String, nil] path to "local DB dir", i.e. folder that contains folders that contain snap.db
+        def local_db_folder(params)
+          Aspera.assert(PARAM_KEYS.any?{ |k| params.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
+          if params.key?('local')
+            # "conf" format
+            if params.key?('local_db_dir')
+              return params['local_db_dir']
+            elsif (local_path = params.dig('local', 'path'))
+              return local_path
+            elsif exception
+              raise Error, 'Missing either local_db_dir or local.path'
+            end
+          else
+            # "args" format
+            session = params['sessions'].first
+            if session.key?('local_db_dir')
+              return session['local_db_dir']
+            elsif session.key?('local_dir')
+              return session['local_dir']
+            elsif exception
+              raise Error, 'Missing either local_db_dir or local_dir'
+            end
+          end
+          nil
+        end
+
+        def session_name(params)
+          Aspera.assert(PARAM_KEYS.any?{ |k| params.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
+          if params.key?('local')
+            # "conf" format
+            return params['name']
+          else
+            # "args" format
+            return params['sessions'].first['name']
+          end
+        end
+
+        def session_db_file(params)
+          db_file = File.join(local_db_folder(params), PRIVATE_FOLDER, session_name(params), ASYNC_DB)
+          Aspera.assert(File.exist?(db_file)){"Database file #{db_file} does not exist"}
+          db_file
+        end
+
+        def list_db_files(local_db_dir)
+          private = File.join(local_db_dir, PRIVATE_FOLDER)
+          Dir.children(private).filter_map do |name|
+            db_file = File.join(private, name, ASYNC_DB)
+            [name, db_file] if File.exist?(db_file)
+          end.to_h
+        end
+
+        # private
+
+        # Transfer specification to synchronization information
+        # tag `x-ts-name` in schema is used to map transfer spec parameters to async `sync_info`
+        # @param transfer_spec [Hash] transfer specification
+        # @param sync_info     [Hash] synchronization information
+        # @param schema        [Hash] schema definition
+        def tspec_to_sync_info(transfer_spec, sync_info, schema)
+          Log.dump(:tspec_to_sync_info, transfer_spec)
+          schema['properties'].each do |name, property|
+            if property.key?('x-ts-name')
+              tspec_param = property['x-ts-name']
+              if transfer_spec.key?(tspec_param) && !sync_info.key?(name)
+                sync_info[name] = property['x-ts-convert'] ? CommandLineConverter.send(property['x-ts-convert'], transfer_spec[tspec_param]) : transfer_spec[tspec_param]
+              end
+            end
+            if property['type'].eql?('object') && property.key?('properties')
+              sync_info[name] ||= {}
+              tspec_to_sync_info(transfer_spec, sync_info[name], property)
+            end
+          end
+        end
+      end
+      # Private stuff:
+      # Read JSON schema and mapping to command line options
+      INSTANCE_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'args')
+      SESSION_SCHEMA = INSTANCE_SCHEMA['properties']['sessions']['items']
+      INSTANCE_SCHEMA['properties'].delete('sessions')
+      CONF_SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'conf')
+      CommandLineBuilder.adjust_properties_defaults(INSTANCE_SCHEMA['properties'])
+      CommandLineBuilder.adjust_properties_defaults(SESSION_SCHEMA['properties'])
+      CommandLineBuilder.adjust_properties_defaults(CONF_SCHEMA['properties'])
+      CMDLINE_PARAMS_KEYS = %w[instance sessions].freeze
+      ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
+      PRIVATE_FOLDER = '.private-asp'
+      ASYNC_DB = 'snap.db'
+      PARAM_KEYS = %w[local sessions].freeze
+
+      private_constant :INSTANCE_SCHEMA, :SESSION_SCHEMA, :CONF_SCHEMA, :CMDLINE_PARAMS_KEYS, :ASYNC_ADMIN_EXECUTABLE, :PRIVATE_FOLDER, :ASYNC_DB, :PARAM_KEYS
+    end
+  end
+end

data/lib/aspera/temp_file_manager.rb

@@ -8,12 +8,13 @@ module Aspera
   # create a temp file name for a given folder
   # files can be deleted on process exit by calling cleanup
   class TempFileManager
+    include Singleton
+
     SEC_IN_DAY = 86_400
     # assume no transfer last longer than this
     # (garbage collect file list which were not deleted after transfer)
     FILE_LIST_AGE_MAX_SEC = SEC_IN_DAY * 5
     private_constant :SEC_IN_DAY, :FILE_LIST_AGE_MAX_SEC
-    include Singleton
 
     attr_accessor :cleanup_on_exit
 
@@ -44,7 +45,7 @@ module Aspera
     end
 
     # same as above but in global temp folder, with user's name
-    def new_file_path_global(prefix=nil, suffix: nil)
+    def new_file_path_global(prefix = nil, suffix: nil)
       username =
         begin
           Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user'

data/lib/aspera/transfer/error.rb

@@ -8,7 +8,7 @@ module Aspera
     class Error < StandardError
       attr_reader :err_code
 
-      def initialize(message, err_code=nil)
+      def initialize(message, err_code = nil)
         super(message)
         @err_code = err_code
       end

data/lib/aspera/transfer/error_info.rb

@@ -6,7 +6,6 @@ module Aspera
   module Transfer
     # from https://www.google.com/search?q=FASP+error+codes
     # Note that the fact that an error is retry-able is not internally defined by protocol, it's client-side responsibility
-    # rubocop:disable Layout/MultilineHashKeyLineBreaks
     # rubocop:disable Layout/FirstHashElementLineBreak
     ERROR_INFO = {
       # id   retry-able     mnemo                       message                                              additional info
@@ -87,6 +86,6 @@ module Aspera
       66 => {r: false, c: 'PEER_REQUIRES_FIPS',          m: 'Peer rejects cipher due to FIPS mode enabled on peer',
                                                               a: 'Peer rejects cipher due to FIPS mode enabled on peer'}
     }.freeze
-    # rubocop:enable Layout/MultilineHashKeyLineBreaks
+    # rubocop:enable Layout/FirstHashElementLineBreak
   end
 end

data/lib/aspera/transfer/faux_file.rb

@@ -2,23 +2,24 @@
 
 module Aspera
   module Transfer
-    # generates a pseudo file stream
+    # Generates a pseudo file stream.
     class FauxFile
+      SCHEME = 'faux'
       # marker for faux file
-      PREFIX = 'faux:///'
-      # size suffix
-      SUFFIX = %w[k m g t p e]
-      private_constant :PREFIX, :SUFFIX
+      PREFIX = "#{SCHEME}:///"
+      # size units, kilo, mega ...
+      SIZE_UNITS = %w[k m g t p e].freeze
+      private_constant :SCHEME, :PREFIX, :SIZE_UNITS
       class << self
         # @return nil if not a faux: scheme, else a FauxFile instance
         def create(name)
-          return nil unless name.start_with?(PREFIX)
+          return unless name.start_with?(PREFIX)
           name_params = name[PREFIX.length..-1].split('?', 2)
-          raise 'Format: #{PREFIX}<file path>?<size>' unless name_params.length.eql?(2)
-          raise "Format: <integer>[#{SUFFIX.join(',')}]" unless (m = name_params[1].downcase.match(/^(\d+)([#{SUFFIX.join('')}])$/))
+          raise Error, 'Format: #{PREFIX}<file path>?<size>' unless name_params.length.eql?(2)
+          raise Error, "Format: <integer>[#{SIZE_UNITS.join(',')}]" unless (m = name_params[1].downcase.match(/^(\d+)([#{SIZE_UNITS.join('')}])$/))
           size = m[1].to_i
           suffix = m[2]
-          SUFFIX.each do |s|
+          SIZE_UNITS.each do |s|
             size *= 1024
             break if s.eql?(suffix)
           end
@@ -36,7 +37,7 @@ module Aspera
       end
 
       def read(chunk_size)
-        return nil if eof?
+        return if eof?
         bytes_to_read = [chunk_size, @size - @offset].min
         @offset += bytes_to_read
         @chunk_by_size[bytes_to_read] = "\x00" * bytes_to_read unless @chunk_by_size.key?(bytes_to_read)

data/lib/aspera/transfer/parameters.rb

@@ -6,7 +6,7 @@ require 'aspera/command_line_builder'
 require 'aspera/temp_file_manager'
 require 'aspera/transfer/error'
 require 'aspera/transfer/spec'
-require 'aspera/transfer/convert'
+require 'aspera/command_line_converter'
 require 'aspera/ascp/installation'
 require 'aspera/cli/formatter'
 require 'aspera/rest'
@@ -20,6 +20,7 @@ module Aspera
   module Transfer
     # translate transfer specification to ascp parameter list
     class Parameters
+      # `ascp` options to provide a file list
       FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
       private_constant :FILE_LIST_OPTIONS
       HTTP_FALLBACK_ACTIVATION_VALUES = ['1', 1, true, 'force'].freeze
@@ -70,7 +71,7 @@ module Aspera
         Aspera.assert(@ascp_args.all?(String)){'all ascp arguments must be String'}
         Aspera.assert_type(@trusted_certs, Array){'trusted_certs'}
         Aspera.assert_values(@client_ssh_key, Ascp::Installation::CLIENT_SSH_KEY_OPTIONS)
-        @builder = CommandLineBuilder.new(@job_spec, Spec::SCHEMA, Convert)
+        @builder = CommandLineBuilder.new(@job_spec, Spec::SCHEMA, CommandLineConverter)
       end
 
       # either place source files on command line, or add file list file
@@ -103,7 +104,7 @@ module Aspera
               lines = ts_paths_array.map{ |i| i['source']}
             end
             file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
-            Log.log.debug{Log.dump(:file_list, lines)}
+            Log.dump(:file_list, lines)
             File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
             Log.log.debug{"#{file_list_option}=\n#{File.read(file_list_file)}".red}
           end
@@ -111,7 +112,7 @@ module Aspera
         @builder.add_command_line_options(["#{file_list_option}=#{file_list_file}"]) unless file_list_option.nil?
       end
 
-      # @return the list of certificates to use when token/ssh or wss are used
+      # @return the list of certificates (option `-i`) to use when token/ssh or wss are used
       def remote_certificates
         certificates_to_use = []
         # use web socket secure for session ?
@@ -139,7 +140,7 @@ module Aspera
           # remove unused parameter (avoid warning)
           @job_spec.delete('wss_port')
           # add SSH bypass keys when authentication is token and no auth is provided
-          if @job_spec.key?('token') && !@job_spec.key?('remote_password')
+          if @job_spec.key?('token') && !@job_spec.key?('remote_password') && !@job_spec.key?('ssh_private_key')
             # @job_spec['remote_password'] = Ascp::Installation.instance.ssh_cert_uuid # not used: no passphrase
             certificates_to_use.concat(Ascp::Installation.instance.aspera_token_ssh_key_paths(@client_ssh_key))
           end

data/lib/aspera/transfer/spec.rb

@@ -25,6 +25,14 @@ module Aspera
       # reserved tag for Aspera
       TAG_RESERVED = 'aspera'
       class << self
+        # wrong def in transferd
+        POLICY_FIX = {
+          'none'        => 'none',
+          'attrs'       => 'attributes',
+          'sparse_csum' => 'sparse_checksum',
+          'full_csum'   => 'full_checksum'
+        }
+        private_constant :POLICY_FIX
         # translate upload/download to send/receive
         def transfer_type_to_direction(transfer_type)
           XFER_TYPE_TO_DIR.fetch(transfer_type)
@@ -34,8 +42,14 @@ module Aspera
         def direction_to_transfer_type(direction)
           XFER_DIR_TO_TYPE.fetch(direction)
         end
+
+        def fix_transferd_resume_policy(transfer_spec)
+          # Fix discrepency in transfer spec
+          transfer_spec['resume_policy'] = POLICY_FIX[transfer_spec['resume_policy']] if transfer_spec.key?('resume_policy')
+        end
       end
-      SCHEMA = CommandLineBuilder.read_schema(__FILE__)
+      SCHEMA = CommandLineBuilder.read_schema(__FILE__, 'spec')
+      CommandLineBuilder.adjust_properties_defaults(SCHEMA['properties'])
       # define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
       SCHEMA['properties'].each do |name, description|
         next unless description['enum'].is_a?(Array)