aspera-cli 4.26.1 → 4.26.2

data/lib/aspera/schema/async_tables.yaml

@@ -0,0 +1,361 @@
+# Schema for the SQLite tables inside the Async snap database (snap.db).
+# Sources:
+#   include/async/session/Sqlite3_SSDB.h  (SYNC_SQLITE3_META_SCHEMA, SYNC_SQLITE3_COUNTERS_SCHEMA, SYNC_SQLITE3_SCHEMA_1_0)
+#   include/async/common/Sync_types.h     (SSSTATE values)
+#   Verified against a live snap.db using: ascli config sync admin overview <path>
+$schema: https://json-schema.org/draft/2020-12/schema
+$id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/schema/async_tables.yaml
+title: Async snap database tables
+description: Schemas for the SQLite tables stored inside the Async snap database (`snap.db`).
+type: object
+properties:
+  meta:
+    title: sync_snapmeta_table
+    description: One row per session, written at start and updated at stop.
+    type: object
+    properties:
+      dbversion:
+        description: Database schema version.
+        type: string
+      pairname:
+        description: Sync session name.
+        type: string
+      ldir:
+        description: Local directory path.
+        type: string
+      rhost:
+        description: Remote host.
+        type: string
+      rdir:
+        description: Remote directory path.
+        type: string
+      direction:
+        description: Sync direction.
+        type: string
+        enum:
+          - push
+          - pull
+          - bidi
+      inode_format:
+        description: Inode format used by the session.
+        type: string
+      checksum:
+        description: Checksum algorithm.
+        type: string
+        enum:
+          - sha1
+          - md5
+          - sha1_sparse
+          - md5_sparse
+          - none
+      start_time:
+        description: Session start timestamp.
+        type: string
+        format: date-time
+      stop_time:
+        description: Session stop timestamp (written when session ends).
+        type: string
+        format: date-time
+      condition:
+        description: Last known session condition.
+        type: string
+        enum:
+          - running
+          - stopped
+      scn_true_val:
+        description: Scan counter true value (internal).
+        type: integer
+      sync_point:
+        description: Synchronization point sequence number.
+        type: integer
+      max_recid:
+        description: Highest record ID allocated in the snapshot table.
+        type: integer
+      sync_uuid:
+        description: UUID identifying this sync pair.
+        type: string
+      cip_true_val:
+        description: Candidate-in-progress counter true value (internal).
+        type: integer
+      exclude_dirs_older_than:
+        description: Exclude directories with modification time older than this epoch value (0 = disabled).
+        type: integer
+      path_synced:
+        description: Number of paths successfully synchronized.
+        type: integer
+      byte_synced:
+        description: Number of bytes successfully synchronized.
+        type: integer
+      filters:
+        description: Active include/exclude filter patterns.
+        type: string
+      path_peerxfer:
+        description: Number of paths transferred by the peer.
+        type: integer
+      byte_peerxfer:
+        description: Number of bytes transferred by the peer.
+        type: integer
+      path_xferres:
+        description: Number of paths for which transfer was resumed.
+        type: integer
+      byte_xferres:
+        description: Number of bytes for which transfer was resumed.
+        type: integer
+      file_synced:
+        description: Number of files successfully synchronized.
+        type: integer
+      file_peerxfer:
+        description: Number of files transferred by the peer.
+        type: integer
+      file_xferres:
+        description: Number of files for which transfer was resumed.
+        type: integer
+      ascp_sids:
+        description: Last known FASP (ascp) session IDs.
+        type: string
+  counters:
+    title: sync_snap_counters_table
+    description: One row, updated live during the session with path and file counts per state.
+    type: object
+    properties:
+      syncd:
+        description: Number of paths in the `Syncd` state.
+        type: integer
+      conflict:
+        description: Number of paths in the `Conflict` state.
+        type: integer
+      potential_conflict:
+        description: Number of paths in the `Potential conflict` state.
+        type: integer
+      error:
+        description: Number of paths in the `Error` state.
+        type: integer
+      pending:
+        description: Number of paths still pending synchronization.
+        type: integer
+      unused:
+        description: Reserved (unused counter slot).
+        type: integer
+      files_syncd:
+        description: Number of files in the `Syncd` state.
+        type: integer
+      files_conflict:
+        description: Number of files in the `Conflict` state.
+        type: integer
+      files_potential_conflict:
+        description: Number of files in the `Potential conflict` state.
+        type: integer
+      files_error:
+        description: Number of files in the `Error` state.
+        type: integer
+      files_pending:
+        description: Number of files still pending synchronization.
+        type: integer
+  file_info:
+    title: sync_snapdb_table
+    description: One row per tracked file or directory. Primary source for the `file_info` command.
+    type: object
+    properties:
+      record_id:
+        description: Primary key (auto-increment).
+        type: integer
+      nodeid:
+        description: Local filesystem node ID.
+        type: integer
+      peer_rid:
+        description: Peer record ID for this entry.
+        type: integer
+      state:
+        description: |
+          Sync state of the entry.
+
+          | Value | Name              |
+          |-------|-------------------|
+          | 0     | Nil               |
+          | 1–18  | Pending (variant) |
+          | 19    | Syncd             |
+          | 20    | Error             |
+          | 21    | Conflict          |
+          | 22    | PotentialConflict |
+          | 23–24 | Pending (variant) |
+        type: integer
+      version:
+        description: Schema version field (not used).
+        type: integer
+      c_meta_type:
+        description: Current metadata — file type (file, directory, …).
+        type: integer
+      c_meta_parentrid:
+        description: Current metadata — parent record ID.
+        type: integer
+      c_meta_path:
+        description: Current metadata — file path as known to the snapshot.
+        type: string
+      c_meta_attrs:
+        description: Current metadata — file attributes.
+        type: string
+      c_meta_uid:
+        description: Current metadata — user ID (Unix).
+        type: integer
+      c_meta_gid:
+        description: Current metadata — group ID (Unix).
+        type: integer
+      c_meta_mtime:
+        description: Current metadata — modification time (epoch).
+        type: integer
+      c_meta_atime:
+        description: Current metadata — access time (epoch).
+        type: integer
+      c_meta_ctime:
+        description: Current metadata — change time (epoch).
+        type: integer
+      c_meta_xattrs:
+        description: Current metadata — extended attributes (not used).
+        type: integer
+      c_meta_xattrs_cksum:
+        description: Current metadata — checksum of extended attributes.
+        type: string
+      c_meta_ol_legal_hold:
+        description: Current metadata — object lock legal hold flag.
+        type: integer
+      c_meta_ol_mode:
+        description: Current metadata — object lock mode.
+        type: integer
+      c_meta_ol_retain_until:
+        description: Current metadata — object lock retain-until date (epoch).
+        type: integer
+      c_meta_object_metadata:
+        description: Current metadata — object metadata (JSON).
+        type: string
+      c_cont_size:
+        description: Current content — file size in bytes.
+        type: integer
+      c_cont_csum:
+        description: Current content — checksum.
+        type: string
+      c_cont_mtime:
+        description: Current content — modification time (epoch).
+        type: integer
+      c_cont_timestamp:
+        description: Current content — internal timestamp.
+        type: integer
+      p_cont_size:
+        description: Pending content — file size in bytes.
+        type: integer
+      p_cont_csum:
+        description: Pending content — checksum.
+        type: string
+      p_cont_mtime:
+        description: Pending content — modification time (epoch).
+        type: integer
+      p_cont_timestamp:
+        description: Pending content — internal timestamp.
+        type: integer
+      f_meta_type:
+        description: Filesystem metadata — file type.
+        type: integer
+      f_meta_parentrid:
+        description: Filesystem metadata — parent record ID.
+        type: integer
+      f_meta_path:
+        description: Filesystem metadata — file path as seen on disk (displayed by `file_info`).
+        type: string
+      f_meta_attrs:
+        description: Filesystem metadata — file attributes.
+        type: string
+      f_meta_uid:
+        description: Filesystem metadata — user ID (Unix).
+        type: integer
+      f_meta_gid:
+        description: Filesystem metadata — group ID (Unix).
+        type: integer
+      f_meta_mtime:
+        description: Filesystem metadata — modification time (epoch).
+        type: integer
+      f_meta_atime:
+        description: Filesystem metadata — access time (epoch).
+        type: integer
+      f_meta_ctime:
+        description: Filesystem metadata — change time (epoch).
+        type: integer
+      f_meta_xattrs:
+        description: Filesystem metadata — extended attributes (not used).
+        type: integer
+      f_meta_xattrs_cksum:
+        description: Filesystem metadata — checksum of extended attributes.
+        type: string
+      f_meta_ol_legal_hold:
+        description: Filesystem metadata — object lock legal hold flag.
+        type: integer
+      f_meta_ol_mode:
+        description: Filesystem metadata — object lock mode.
+        type: integer
+      f_meta_ol_retain_until:
+        description: Filesystem metadata — object lock retain-until date (epoch).
+        type: integer
+      f_meta_object_metadata:
+        description: Filesystem metadata — object metadata (JSON).
+        type: string
+      f_cont_size:
+        description: Filesystem content — file size in bytes.
+        type: integer
+      f_cont_csum:
+        description: Filesystem content — checksum.
+        type: string
+      f_cont_mtime:
+        description: Filesystem content — modification time (epoch).
+        type: integer
+      f_cont_timestamp:
+        description: Filesystem content — internal timestamp.
+        type: integer
+      pcr_reqno:
+        description: Peer content request sequence number.
+        type: integer
+      pp_cont_size:
+        description: Peer pending content — file size in bytes.
+        type: integer
+      pp_cont_csum:
+        description: Peer pending content — checksum.
+        type: string
+      pp_cont_mtime:
+        description: Peer pending content — modification time (epoch).
+        type: integer
+      pp_cont_timestamp:
+        description: Peer pending content — internal timestamp.
+        type: integer
+      tobe_deleted:
+        description: Set to 1 when the record is flagged for deletion (garbage collection pending).
+        type: integer
+      scn_not_visited:
+        description: Set to 1 when the scanner has not yet visited this entry in the current scan pass.
+        type: integer
+      scn_required:
+        description: Set to 1 when a rescan of this entry is required.
+        type: integer
+      scn_completed:
+        description: Set to 1 when the scan of this directory entry has completed.
+        type: integer
+      candidate_in_prog:
+        description: Set to 1 when this record is currently in the candidate transfer list.
+        type: integer
+      meta_lastreq_no:
+        description: Sequence number of the last metadata request sent.
+        type: integer
+      meta_lastresp_no:
+        description: Sequence number of the last metadata response received.
+        type: integer
+      commit_version:
+        description: Local commit version for this entry.
+        type: integer
+      peer_commit_version:
+        description: Peer commit version for this entry.
+        type: integer
+      peer_nodeid:
+        description: Peer filesystem node ID.
+        type: integer
+      recursive_mtime:
+        description: Most recent modification time across all descendants (directories only).
+        type: integer
+      message:
+        description: Error or status message associated with this entry.
+        type: string

data/lib/aspera/schema/reader.rb

@@ -24,7 +24,7 @@ module Aspera
           Aspera.assert_type(current, Hash){'schema'}
           if current.key?('$ref')
             ref = current['$ref']
-            Aspera.assert(ref.start_with?('#/'))
+            Aspera.assert(ref.start_with?('#/')){"schema $ref must start with '#/': #{ref}"}
             current = @root.dig(*ref[2..].split('/'))
           end
         end

data/lib/aspera/schema/registry.rb

@@ -7,6 +7,9 @@ require 'aspera/schema/reader'
 module Aspera
   # base class for plugins modules
   module Schema
+    # @!method self.instance
+    #   Returns the singleton instance of Registry
+    #   @return [Registry] the singleton instance
     class Registry
       include Singleton
 
@@ -15,18 +18,21 @@ module Aspera
           LOCATIONS.key?(sym)
         end
 
+        # Get path to request body, no check if it exists
+        # @return [String] path to request body
         def req_body(component, endpoint)
           "#{component}:paths./#{endpoint}.requestBody.content.application/json.schema"
         end
       end
 
       LOCATIONS = {
-        spec:   'aspera/transfer/spec.schema.yaml',
-        args:   'aspera/sync/args.schema.yaml',
-        conf:   'aspera/sync/conf.schema.yaml',
-        opts:   'aspera/cli/options.schema.yaml',
-        aoc:    'aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml',
-        faspex: 'aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml'
+        spec:         'aspera/transfer/spec.schema.yaml',
+        args:         'aspera/sync/args.schema.yaml',
+        conf:         'aspera/sync/conf.schema.yaml',
+        opts:         'aspera/cli/options.schema.yaml',
+        aoc:          'aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml',
+        faspex:       'aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml',
+        async_tables: 'aspera/schema/async_tables.yaml'
       }
 
       OPTIONS = 'opts'
@@ -35,6 +41,7 @@ module Aspera
       SYNC_ARGS = 'args'
       AOC = 'aoc'
       FASPEX = 'faspex'
+      ASYNC_TABLES = 'async_tables'
       TRANSFER_INFO = "#{OPTIONS}:components.schemas.TransferInfo"
 
       REQ_BODY = '.requestBody.content.application/json.schema'

data/lib/aspera/secret_hider.rb

@@ -6,6 +6,10 @@ require 'singleton'
 
 module Aspera
   # remove secret from logs and output
+  #
+  # @!method self.instance
+  #   Returns the singleton instance of SecretHider
+  #   @return [SecretHider] the singleton instance
   class SecretHider
     include Singleton
 

data/lib/aspera/sync/database.rb

@@ -35,12 +35,12 @@ class SqLite3Wrapper
   end
 
   # The table contains a single row
-  def single_table(table_name)
-    execute("SELECT * FROM #{table_name} LIMIT 1").first
+  def single_table(table_name, sql_suffix = nil)
+    execute(["SELECT * FROM #{table_name}", sql_suffix, 'LIMIT 1'].compact.join(' ')).first
   end
 
-  def full_table(table_name)
-    execute("SELECT * FROM #{table_name}")
+  def full_table(table_name, sql_suffix = nil)
+    execute(["SELECT * FROM #{table_name}", sql_suffix].compact.join(' '))
   end
 
   private
@@ -68,16 +68,20 @@ module Aspera
         end
       end
 
-      def meta
-        @db.single_table('sync_snapmeta_table')
+      def meta(sql_suffix = nil)
+        @db.single_table('sync_snapmeta_table', sql_suffix)
       end
 
-      def counters
-        @db.single_table('sync_snap_counters_table')
+      def counters(sql_suffix = nil)
+        @db.single_table('sync_snap_counters_table', sql_suffix)
       end
 
-      def file_info
-        @db.full_table('sync_snapdb_table')
+      def file_info(sql_suffix = nil)
+        @db.full_table('sync_snapdb_table', sql_suffix)
+      end
+
+      def execute(sql)
+        @db.execute(sql)
       end
     end
   end

data/lib/aspera/sync/operations.rb

@@ -89,7 +89,7 @@ module Aspera
         def start(sync_info, opt_ts = nil)
           Log.dump(:sync_params_initial, sync_info)
           Aspera.assert_type(sync_info, Hash)
-          Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
+          Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
           env_args = {
             args: [],
             env:  {}
@@ -145,7 +145,7 @@ module Aspera
             end
             sync_info['sessions'].each do |session_params|
               Aspera.assert_type(session_params, Hash)
-              Aspera.assert(session_params.key?('name')){'session must contain at least: name'}
+              Aspera.assert(session_params.key?('name'), 'session must contain at least: name')
               session_builder = CommandLineBuilder.new(session_params, ARGS_SESSION_SCHEMA, CommandLineConverter)
               session_builder.process_params
               session_builder.add_env_args(env_args)
@@ -177,7 +177,7 @@ module Aspera
         # @param sync_info [Hash] sync parameters in conf or args format
         # @return [Hash] parsed output of asyncadmin
         def admin_status(sync_info)
-          Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
+          Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
           arguments = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
           if sync_info.key?('local')
             # `conf` format
@@ -209,7 +209,7 @@ module Aspera
         # @param sync_info [Hash] sync parameters in conf or args format
         # @return [String, nil] Path to "local DB dir", i.e. folder that contains folders that contain `snap.db`
         def local_db_folder(sync_info)
-          Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
+          Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
           if sync_info.key?('local')
             # `conf` format
             if sync_info.key?('local_db_dir')
@@ -234,7 +234,7 @@ module Aspera
         end
 
         def session_name(sync_info)
-          Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
+          Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, 'At least one of `local` or `sessions` must be present in async parameters', type: Error)
           if sync_info.key?('local')
             # `conf` format
             return sync_info['name']

data/lib/aspera/temp_file_manager.rb

@@ -7,6 +7,10 @@ require 'etc'
 module Aspera
   # create a temp file name for a given folder
   # files can be deleted on process exit by calling cleanup
+  #
+  # @!method self.instance
+  #   Returns the singleton instance of TempFileManager
+  #   @return [TempFileManager] the singleton instance
   class TempFileManager
     include Singleton
 

data/lib/aspera/transfer/parameters.rb

@@ -85,12 +85,12 @@ module Aspera
         file_list_option = nil
         # transfer spec contains paths ?
         if !ts_paths_array.nil?
-          Aspera.assert(!ascp_file_list_provided){'file list provided both in transfer spec and ascp file list. Remove one of them.'}
-          Aspera.assert(ts_paths_array.all?{ |i| i.key?('source')}){"All elements of paths must have a 'source' key"}
+          Aspera.assert(!ascp_file_list_provided, 'file list provided both in transfer spec and ascp file list. Remove one of them.')
+          Aspera.assert(ts_paths_array.all?{ |i| i.key?('source')}, "All elements of paths must have a 'source' key")
           is_pair_list = ts_paths_array.any?{ |i| i.key?('destination')}
           raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{ |i| i.key?('destination')}
           if self.class.file_list_folder.nil?
-            Aspera.assert(!is_pair_list){'file pair list is not supported when file list folder is not set'}
+            Aspera.assert(!is_pair_list, 'file pair list is not supported when file list folder is not set')
             # not safe for special characters ? (maybe not, depends on OS)
             Log.log.debug('placing source file list on command line (no file list file)')
             @builder.add_command_line_options(ts_paths_array.map{ |i| i['source']})
@@ -160,7 +160,7 @@ module Aspera
         @job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
 
         # Notify multi-session was already used, anyway it was deleted by agent direct
-        Aspera.assert(!@builder.read_param('multi_session'))
+        Aspera.assert(!@builder.read_param('multi_session'), 'multi_session already consumed, must not be set here')
 
         # Add ssh or wss certificates
         # (reverse, to keep order, as we unshift)

data/lib/aspera/transfer/resumer.rb

@@ -34,7 +34,7 @@ module Aspera
       #
       # @yieldreturn [void] Executes the transfer operation
       def execute_with_resume
-        Aspera.assert(block_given?)
+        Aspera.assert(block_given?, 'block required for execute_with_resume')
         # maximum of retry
         remaining_resumes = @iter_max
         sleep_seconds = @sleep_initial

data/lib/aspera/transfer/spec.schema.yaml

@@ -706,7 +706,17 @@ properties:
       - transferd
     x-cli-option: -i
   symlink_policy:
-    description: Handle source side symbolic links.
+    description: >-
+      Handle source side symbolic links.
+      Refer to HSTS manual for more details.
+
+      - `follow`: Follow a symbolic link and transfer the contents of the linked file or directory when the link target is in the user's docroot.
+      
+      - `copy`: Copy only the symbolic link. If a file with the same name exists at the destination, the symbolic link does not replace the file.
+      
+      - `skip`: Skip-symbolic links. The link or the file to which it points are not transferred.
+      
+      - `copy+force`: Copy only the symbolic link. If a file with the same name exists at the destination, the symbolic link replaces the file. If the file of the same name at the destination is a symbolic link to a directory, it is not replaced.
     type: string
     enum:
       - follow
@@ -719,15 +729,13 @@ properties:
     description: >-
       Metadata for transfer as JSON.
       Key `aspera` is reserved.
-      Key `aspera.xfer_retry` specifies a retry timeout for node API initiated transfers.
+      Key `aspera.xfer_retry` specifies a "retry timeout" but is not actually used for node API initiated transfers.
     type: object
     x-cli-option: --tags64
     x-cli-convert: json64
   tags64:
     description: >-
-      Metadata for transfer as JSON.
-      Key `aspera` is reserved.
-      Key `aspera.xfer_retry` specifies a retry timeout for node API initiated transfers.
+      Base64 string for `tags`.
     type: string
     x-agents:
       - direct
@@ -753,11 +761,12 @@ properties:
       - connect
       - transferd
   token:
-    description: "Authorization token. Type: Bearer, Basic or ATM. (Also arg -W)"
+    $comment: Also arg -W
+    description: Authorization token. Type is Bearer, Basic or ATM.
     type: string
     x-cli-envvar: ASPERA_SCP_TOKEN
   use_ascp4:
-    description: Specify version of protocol. Do not use `ascp4`.
+    description: Specify version of protocol. `ascp4` will be deprecated.
     type: boolean
     default: false
     x-agents:
@@ -881,7 +890,7 @@ properties:
   xfer_max_retries:
     description: >-
       Maximum number of retries, for node API initiated transfers.
-      Shall not exceed `aspera.conf` parameter `transfer_manager_max_retries` (default 5).
+      Shall not exceed `aspera.conf` parameter `transfer_manager_max_retries` (default 3).
     type: integer
     x-agents:
       - node

data/lib/aspera/transfer/uri.rb

@@ -13,7 +13,7 @@ module Aspera
       SCHEME = 'faspe'
       def initialize(fasp_link)
         @fasp_uri = URI.parse(fasp_link.gsub(' ', '%20'))
-        Aspera.assert(@fasp_uri.scheme == SCHEME, "Invalid scheme: #{@fasp_uri.scheme}")
+        Aspera.assert_values(@fasp_uri.scheme, [SCHEME]){"Invalid scheme: #{@fasp_uri.scheme}"}
       end
 
       # Generate transfer spec from provided faspe: URL