aspera-cli 4.26.0 → 4.26.2

data/lib/aspera/schema/async_tables.yaml

@@ -0,0 +1,361 @@
+# Schema for the SQLite tables inside the Async snap database (snap.db).
+# Sources:
+#   include/async/session/Sqlite3_SSDB.h  (SYNC_SQLITE3_META_SCHEMA, SYNC_SQLITE3_COUNTERS_SCHEMA, SYNC_SQLITE3_SCHEMA_1_0)
+#   include/async/common/Sync_types.h     (SSSTATE values)
+#   Verified against a live snap.db using: ascli config sync admin overview <path>
+$schema: https://json-schema.org/draft/2020-12/schema
+$id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/schema/async_tables.yaml
+title: Async snap database tables
+description: Schemas for the SQLite tables stored inside the Async snap database (`snap.db`).
+type: object
+properties:
+  meta:
+    title: sync_snapmeta_table
+    description: One row per session, written at start and updated at stop.
+    type: object
+    properties:
+      dbversion:
+        description: Database schema version.
+        type: string
+      pairname:
+        description: Sync session name.
+        type: string
+      ldir:
+        description: Local directory path.
+        type: string
+      rhost:
+        description: Remote host.
+        type: string
+      rdir:
+        description: Remote directory path.
+        type: string
+      direction:
+        description: Sync direction.
+        type: string
+        enum:
+          - push
+          - pull
+          - bidi
+      inode_format:
+        description: Inode format used by the session.
+        type: string
+      checksum:
+        description: Checksum algorithm.
+        type: string
+        enum:
+          - sha1
+          - md5
+          - sha1_sparse
+          - md5_sparse
+          - none
+      start_time:
+        description: Session start timestamp.
+        type: string
+        format: date-time
+      stop_time:
+        description: Session stop timestamp (written when session ends).
+        type: string
+        format: date-time
+      condition:
+        description: Last known session condition.
+        type: string
+        enum:
+          - running
+          - stopped
+      scn_true_val:
+        description: Scan counter true value (internal).
+        type: integer
+      sync_point:
+        description: Synchronization point sequence number.
+        type: integer
+      max_recid:
+        description: Highest record ID allocated in the snapshot table.
+        type: integer
+      sync_uuid:
+        description: UUID identifying this sync pair.
+        type: string
+      cip_true_val:
+        description: Candidate-in-progress counter true value (internal).
+        type: integer
+      exclude_dirs_older_than:
+        description: Exclude directories with modification time older than this epoch value (0 = disabled).
+        type: integer
+      path_synced:
+        description: Number of paths successfully synchronized.
+        type: integer
+      byte_synced:
+        description: Number of bytes successfully synchronized.
+        type: integer
+      filters:
+        description: Active include/exclude filter patterns.
+        type: string
+      path_peerxfer:
+        description: Number of paths transferred by the peer.
+        type: integer
+      byte_peerxfer:
+        description: Number of bytes transferred by the peer.
+        type: integer
+      path_xferres:
+        description: Number of paths for which transfer was resumed.
+        type: integer
+      byte_xferres:
+        description: Number of bytes for which transfer was resumed.
+        type: integer
+      file_synced:
+        description: Number of files successfully synchronized.
+        type: integer
+      file_peerxfer:
+        description: Number of files transferred by the peer.
+        type: integer
+      file_xferres:
+        description: Number of files for which transfer was resumed.
+        type: integer
+      ascp_sids:
+        description: Last known FASP (ascp) session IDs.
+        type: string
+  counters:
+    title: sync_snap_counters_table
+    description: One row, updated live during the session with path and file counts per state.
+    type: object
+    properties:
+      syncd:
+        description: Number of paths in the `Syncd` state.
+        type: integer
+      conflict:
+        description: Number of paths in the `Conflict` state.
+        type: integer
+      potential_conflict:
+        description: Number of paths in the `Potential conflict` state.
+        type: integer
+      error:
+        description: Number of paths in the `Error` state.
+        type: integer
+      pending:
+        description: Number of paths still pending synchronization.
+        type: integer
+      unused:
+        description: Reserved (unused counter slot).
+        type: integer
+      files_syncd:
+        description: Number of files in the `Syncd` state.
+        type: integer
+      files_conflict:
+        description: Number of files in the `Conflict` state.
+        type: integer
+      files_potential_conflict:
+        description: Number of files in the `Potential conflict` state.
+        type: integer
+      files_error:
+        description: Number of files in the `Error` state.
+        type: integer
+      files_pending:
+        description: Number of files still pending synchronization.
+        type: integer
+  file_info:
+    title: sync_snapdb_table
+    description: One row per tracked file or directory. Primary source for the `file_info` command.
+    type: object
+    properties:
+      record_id:
+        description: Primary key (auto-increment).
+        type: integer
+      nodeid:
+        description: Local filesystem node ID.
+        type: integer
+      peer_rid:
+        description: Peer record ID for this entry.
+        type: integer
+      state:
+        description: |
+          Sync state of the entry.
+
+          | Value | Name              |
+          |-------|-------------------|
+          | 0     | Nil               |
+          | 1–18  | Pending (variant) |
+          | 19    | Syncd             |
+          | 20    | Error             |
+          | 21    | Conflict          |
+          | 22    | PotentialConflict |
+          | 23–24 | Pending (variant) |
+        type: integer
+      version:
+        description: Schema version field (not used).
+        type: integer
+      c_meta_type:
+        description: Current metadata — file type (file, directory, …).
+        type: integer
+      c_meta_parentrid:
+        description: Current metadata — parent record ID.
+        type: integer
+      c_meta_path:
+        description: Current metadata — file path as known to the snapshot.
+        type: string
+      c_meta_attrs:
+        description: Current metadata — file attributes.
+        type: string
+      c_meta_uid:
+        description: Current metadata — user ID (Unix).
+        type: integer
+      c_meta_gid:
+        description: Current metadata — group ID (Unix).
+        type: integer
+      c_meta_mtime:
+        description: Current metadata — modification time (epoch).
+        type: integer
+      c_meta_atime:
+        description: Current metadata — access time (epoch).
+        type: integer
+      c_meta_ctime:
+        description: Current metadata — change time (epoch).
+        type: integer
+      c_meta_xattrs:
+        description: Current metadata — extended attributes (not used).
+        type: integer
+      c_meta_xattrs_cksum:
+        description: Current metadata — checksum of extended attributes.
+        type: string
+      c_meta_ol_legal_hold:
+        description: Current metadata — object lock legal hold flag.
+        type: integer
+      c_meta_ol_mode:
+        description: Current metadata — object lock mode.
+        type: integer
+      c_meta_ol_retain_until:
+        description: Current metadata — object lock retain-until date (epoch).
+        type: integer
+      c_meta_object_metadata:
+        description: Current metadata — object metadata (JSON).
+        type: string
+      c_cont_size:
+        description: Current content — file size in bytes.
+        type: integer
+      c_cont_csum:
+        description: Current content — checksum.
+        type: string
+      c_cont_mtime:
+        description: Current content — modification time (epoch).
+        type: integer
+      c_cont_timestamp:
+        description: Current content — internal timestamp.
+        type: integer
+      p_cont_size:
+        description: Pending content — file size in bytes.
+        type: integer
+      p_cont_csum:
+        description: Pending content — checksum.
+        type: string
+      p_cont_mtime:
+        description: Pending content — modification time (epoch).
+        type: integer
+      p_cont_timestamp:
+        description: Pending content — internal timestamp.
+        type: integer
+      f_meta_type:
+        description: Filesystem metadata — file type.
+        type: integer
+      f_meta_parentrid:
+        description: Filesystem metadata — parent record ID.
+        type: integer
+      f_meta_path:
+        description: Filesystem metadata — file path as seen on disk (displayed by `file_info`).
+        type: string
+      f_meta_attrs:
+        description: Filesystem metadata — file attributes.
+        type: string
+      f_meta_uid:
+        description: Filesystem metadata — user ID (Unix).
+        type: integer
+      f_meta_gid:
+        description: Filesystem metadata — group ID (Unix).
+        type: integer
+      f_meta_mtime:
+        description: Filesystem metadata — modification time (epoch).
+        type: integer
+      f_meta_atime:
+        description: Filesystem metadata — access time (epoch).
+        type: integer
+      f_meta_ctime:
+        description: Filesystem metadata — change time (epoch).
+        type: integer
+      f_meta_xattrs:
+        description: Filesystem metadata — extended attributes (not used).
+        type: integer
+      f_meta_xattrs_cksum:
+        description: Filesystem metadata — checksum of extended attributes.
+        type: string
+      f_meta_ol_legal_hold:
+        description: Filesystem metadata — object lock legal hold flag.
+        type: integer
+      f_meta_ol_mode:
+        description: Filesystem metadata — object lock mode.
+        type: integer
+      f_meta_ol_retain_until:
+        description: Filesystem metadata — object lock retain-until date (epoch).
+        type: integer
+      f_meta_object_metadata:
+        description: Filesystem metadata — object metadata (JSON).
+        type: string
+      f_cont_size:
+        description: Filesystem content — file size in bytes.
+        type: integer
+      f_cont_csum:
+        description: Filesystem content — checksum.
+        type: string
+      f_cont_mtime:
+        description: Filesystem content — modification time (epoch).
+        type: integer
+      f_cont_timestamp:
+        description: Filesystem content — internal timestamp.
+        type: integer
+      pcr_reqno:
+        description: Peer content request sequence number.
+        type: integer
+      pp_cont_size:
+        description: Peer pending content — file size in bytes.
+        type: integer
+      pp_cont_csum:
+        description: Peer pending content — checksum.
+        type: string
+      pp_cont_mtime:
+        description: Peer pending content — modification time (epoch).
+        type: integer
+      pp_cont_timestamp:
+        description: Peer pending content — internal timestamp.
+        type: integer
+      tobe_deleted:
+        description: Set to 1 when the record is flagged for deletion (garbage collection pending).
+        type: integer
+      scn_not_visited:
+        description: Set to 1 when the scanner has not yet visited this entry in the current scan pass.
+        type: integer
+      scn_required:
+        description: Set to 1 when a rescan of this entry is required.
+        type: integer
+      scn_completed:
+        description: Set to 1 when the scan of this directory entry has completed.
+        type: integer
+      candidate_in_prog:
+        description: Set to 1 when this record is currently in the candidate transfer list.
+        type: integer
+      meta_lastreq_no:
+        description: Sequence number of the last metadata request sent.
+        type: integer
+      meta_lastresp_no:
+        description: Sequence number of the last metadata response received.
+        type: integer
+      commit_version:
+        description: Local commit version for this entry.
+        type: integer
+      peer_commit_version:
+        description: Peer commit version for this entry.
+        type: integer
+      peer_nodeid:
+        description: Peer filesystem node ID.
+        type: integer
+      recursive_mtime:
+        description: Most recent modification time across all descendants (directories only).
+        type: integer
+      message:
+        description: Error or status message associated with this entry.
+        type: string

data/lib/aspera/schema/documentation.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'aspera/agent/factory'
+require 'aspera/markdown'
+
+module Aspera
+  module Schema
+    # Generate documentation from Schema, for Transfer Spec, or async Conf spec
+    class Documentation
+      # @param formatter [Cli::Formatter] Formatter instance with methods: markdown_text, tick, check_row
+      # @param schema [Reader]
+      # @param include_option [Boolean] `true`: include CLI options (switches, env vars) in descriptions
+      # @param agent_columns  [Boolean] `true`: add separate columns for each transfer agent compatibility
+      # @param code_highlight [Boolean] `true`: format name and type as code
+      def initialize(formatter, schema, include_option: false, agent_columns: false, code_highlight: false)
+        @formatter = formatter
+        @schema = schema
+        @include_option = include_option
+        @agent_columns = agent_columns
+        @code_highlight = code_highlight
+        @columns = %i[name type description]
+        @columns.insert(-2, *Agent::Factory::ALL.values.map{ |i| i[:short]}.sort) if @agent_columns
+        # @type [Array<Hash<Symbol,String>>]
+        @rows = []
+      end
+
+      def rows
+        @rows.sort_by{ |i| i[:name]}
+      end
+
+      # @return [Array<String>]
+      def columns
+        @columns.map(&:to_s)
+      end
+
+      # First row is the titles
+      # @return [Array<Array<String>>]
+      def table
+        [@columns.map(&:to_s)] + @rows.sort_by{ |i| i[:name]}.map{ |row| @columns.map{ |field| row[field]}}
+      end
+
+      # Generate a documentation table from a JSON schema for transfer specifications
+      #
+      # Recursively processes a JSON schema to create a formatted table for manual documentation.
+      # Handles nested objects, arrays, and extracts metadata (descriptions, types, enums, deprecations).
+      #
+      # @param schema [Reader] The JSON schema to process
+      # @return [Documentation]
+      def build(schema = nil)
+        code = @code_highlight ? ->(c){"`#{c}`"} : ->(c){c}
+        schema ||= @schema
+        schema.each_property do |property_schema, _name, property_full_name|
+          node = property_schema.current
+          # Manual table
+          item = {
+            name:        code.call(property_full_name),
+            type:        code.call(node['type']),
+            description: []
+          }
+          # Render Markdown formatting and split lines
+          item[:description] =
+            node['description']
+              .gsub(Markdown::FORMATS){@formatter.markdown_text(Regexp.last_match)}
+              .split("\n") if node.key?('description')
+          item[:description].unshift("DEPRECATED: #{node['x-deprecation']}") if node.key?('x-deprecation')
+          # Add flags for supported agents in doc
+          agents = []
+          Agent::Factory::ALL.each_key do |sym|
+            agents.push(sym) if node['x-agents'].nil? || node['x-agents'].include?(sym.to_s)
+          end
+          Aspera.assert(agents.include?(:direct)){"#{name}: x-cli-option requires agent direct (or nil)"} if node['x-cli-option']
+          if @agent_columns
+            Agent::Factory::ALL.each do |sym, names|
+              item[names[:short]] = @formatter.tick(agents.include?(sym))
+            end
+          else
+            item[:description].push("(#{agents.map{ |i| Agent::Factory::ALL[i][:short].to_s.upcase}.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
+          end
+          # Only keep lines that are usable in supported agents
+          next false if agents.empty?
+          item[:description].push("Allowed values: #{node['enum'].map{ |v| @formatter.markdown_text("`#{v}`")}.join(', ')}.") if node.key?('enum')
+          item[:description].push("Default: #{code.call(node['default'])}.") if node.key?('default')
+          if @include_option
+            envvar_prefix = ''
+            cli_option =
+              if node.key?('x-cli-envvar')
+                envvar_prefix = 'env:'
+                node['x-cli-envvar']
+              elsif node['x-cli-switch']
+                node['x-cli-option']
+              elsif node['x-cli-option']
+                arg_type = node.key?('enum') ? '{enum}' : "{#{[node['type']].flatten.join('|')}}"
+                # conversion_tag = node['x-cli-convert']
+                conversion_tag = node.key?('x-cli-convert') ? 'conversion' : nil
+                sep = node['x-cli-option'].start_with?('--') ? '=' : ' '
+                "#{node['x-cli-option']}#{sep}#{"(#{conversion_tag})" if conversion_tag}#{arg_type}"
+              end
+            short = node.key?('x-cli-short') ? "(#{node['x-cli-short']})" : nil
+            item[:description].push("(#{'special:' if node['x-cli-special']}#{envvar_prefix}#{@formatter.markdown_text("`#{cli_option}`")})#{short}") if cli_option
+          end
+          @rows.push(@formatter.check_row(item))
+        end
+        self
+      end
+    end
+  end
+end

data/lib/aspera/schema/reader.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Aspera
+  # base class for plugins modules
+  module Schema
+    # JSON schema reader
+    class Reader
+      attr_reader :current
+
+      # Shortcut to access current value at path
+      # @param x [String] path element
+      # @return [Hash, Array, String, Integer] current value at path
+      def [](x)
+        @current[x]
+      end
+
+      # Find sub path relative to current
+      # Honors $ref
+      def dig(*path)
+        current = @current
+        path.each do |p|
+          Aspera.assert(current.key?(p)){"schema: #{p} in #{path}"}
+          current = current[p]
+          Aspera.assert_type(current, Hash){'schema'}
+          if current.key?('$ref')
+            ref = current['$ref']
+            Aspera.assert(ref.start_with?('#/')){"schema $ref must start with '#/': #{ref}"}
+            current = @root.dig(*ref[2..].split('/'))
+          end
+        end
+        Reader.new(@root, current)
+      end
+
+      # Read schema from file or from cache
+      # @param root [Hash] root schema
+      # @param current [Hash, nil] current position in
+      # @return [Hash, nil] schema
+      def initialize(root, current = nil)
+        @root = root
+        @current = current || root
+      end
+
+      # Recursively traverse schema properties with a block
+      # Handles nested objects and arrays automatically
+      # @param prefix [String] Prefix for property names (e.g., 'parent.child.')
+      # @yield [property_schema, name, full_name] Yields property info to block
+      # @yieldparam property_schema [Reader] Schema reader for this property (use .current to get node hash)
+      # @yieldparam name [String] Property name
+      # @yieldparam full_name [String] Full property name with prefix
+      # @return [nil]
+      def each_property(prefix = '', &block)
+        properties = dig('properties')
+        properties.current.each_key do |name|
+          property_full_name = "#{prefix}#{name}"
+          property_schema = properties.dig(name)
+          node = property_schema.current
+
+          # Yield current property to block
+          yield(property_schema, name, property_full_name)
+
+          # Recursively process nested structures
+          case node['type']
+          when 'object'
+            property_schema.each_property("#{property_full_name}.", &block) if node['properties']
+          when 'array'
+            if node['items']
+              array_item_schema = property_schema.dig('items')
+              array_item_schema.each_property("#{property_full_name}[].", &block) if array_item_schema.current['properties']
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aspera/schema/registry.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'singleton'
+require 'aspera/yaml'
+require 'aspera/schema/reader'
+
+module Aspera
+  # base class for plugins modules
+  module Schema
+    # @!method self.instance
+    #   Returns the singleton instance of Registry
+    #   @return [Registry] the singleton instance
+    class Registry
+      include Singleton
+
+      class << self
+        def known?(sym)
+          LOCATIONS.key?(sym)
+        end
+
+        # Get path to request body, no check if it exists
+        # @return [String] path to request body
+        def req_body(component, endpoint)
+          "#{component}:paths./#{endpoint}.requestBody.content.application/json.schema"
+        end
+      end
+
+      LOCATIONS = {
+        spec:         'aspera/transfer/spec.schema.yaml',
+        args:         'aspera/sync/args.schema.yaml',
+        conf:         'aspera/sync/conf.schema.yaml',
+        opts:         'aspera/cli/options.schema.yaml',
+        aoc:          'aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml',
+        faspex:       'aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml',
+        async_tables: 'aspera/schema/async_tables.yaml'
+      }
+
+      OPTIONS = 'opts'
+      TRANSFER_SPEC = 'spec'
+      SYNC_CONF = 'conf'
+      SYNC_ARGS = 'args'
+      AOC = 'aoc'
+      FASPEX = 'faspex'
+      ASYNC_TABLES = 'async_tables'
+      TRANSFER_INFO = "#{OPTIONS}:components.schemas.TransferInfo"
+
+      REQ_BODY = '.requestBody.content.application/json.schema'
+
+      def initialize
+        @cache = {}
+        @main_folder = File.expand_path('../..', __dir__)
+      end
+
+      # Read schema from file or from cache
+      # @param name_path [String] one of the keys in LOCATIONS, with optional :<path> suffix
+      # @return [Reader] schema
+      def reader(name_path)
+        name, path = name_path.split(':', 2)
+        sym = name.to_sym
+        Aspera.assert(Registry.known?(sym)){"schema: #{sym}"}
+        spec_file = File.join(@main_folder, LOCATIONS[sym])
+        @cache[sym] = Yaml.safe_load(File.read(spec_file)) if spec_file.end_with?('.yaml') && !@cache.key?(sym)
+        @cache[sym] = JSON.parse(File.read(spec_file)) if spec_file.end_with?('.json') && !@cache.key?(sym)
+        reader = Reader.new(@cache[sym])
+        return reader unless path
+        reader.dig(*path.split('.'))
+      end
+    end
+  end
+end

data/lib/aspera/secret_hider.rb

@@ -6,6 +6,10 @@ require 'singleton'
 
 module Aspera
   # remove secret from logs and output
+  #
+  # @!method self.instance
+  #   Returns the singleton instance of SecretHider
+  #   @return [SecretHider] the singleton instance
   class SecretHider
     include Singleton