aspera-cli 4.26.1 → 4.27.0

data/lib/aspera/schema/async_tables.yaml

@@ -0,0 +1,361 @@
+# Schema for the SQLite tables inside the Async snap database (snap.db).
+# Sources:
+#   include/async/session/Sqlite3_SSDB.h  (SYNC_SQLITE3_META_SCHEMA, SYNC_SQLITE3_COUNTERS_SCHEMA, SYNC_SQLITE3_SCHEMA_1_0)
+#   include/async/common/Sync_types.h     (SSSTATE values)
+#   Verified against a live snap.db using: ascli config sync admin overview <path>
+$schema: https://json-schema.org/draft/2020-12/schema
+$id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/schema/async_tables.yaml
+title: Async snap database tables
+description: Schemas for the SQLite tables stored inside the Async snap database (`snap.db`).
+type: object
+properties:
+  meta:
+    title: sync_snapmeta_table
+    description: One row per session, written at start and updated at stop.
+    type: object
+    properties:
+      dbversion:
+        description: Database schema version.
+        type: string
+      pairname:
+        description: Sync session name.
+        type: string
+      ldir:
+        description: Local directory path.
+        type: string
+      rhost:
+        description: Remote host.
+        type: string
+      rdir:
+        description: Remote directory path.
+        type: string
+      direction:
+        description: Sync direction.
+        type: string
+        enum:
+          - push
+          - pull
+          - bidi
+      inode_format:
+        description: Inode format used by the session.
+        type: string
+      checksum:
+        description: Checksum algorithm.
+        type: string
+        enum:
+          - sha1
+          - md5
+          - sha1_sparse
+          - md5_sparse
+          - none
+      start_time:
+        description: Session start timestamp.
+        type: string
+        format: date-time
+      stop_time:
+        description: Session stop timestamp (written when session ends).
+        type: string
+        format: date-time
+      condition:
+        description: Last known session condition.
+        type: string
+        enum:
+          - running
+          - stopped
+      scn_true_val:
+        description: Scan counter true value (internal).
+        type: integer
+      sync_point:
+        description: Synchronization point sequence number.
+        type: integer
+      max_recid:
+        description: Highest record ID allocated in the snapshot table.
+        type: integer
+      sync_uuid:
+        description: UUID identifying this sync pair.
+        type: string
+      cip_true_val:
+        description: Candidate-in-progress counter true value (internal).
+        type: integer
+      exclude_dirs_older_than:
+        description: Exclude directories with modification time older than this epoch value (0 = disabled).
+        type: integer
+      path_synced:
+        description: Number of paths successfully synchronized.
+        type: integer
+      byte_synced:
+        description: Number of bytes successfully synchronized.
+        type: integer
+      filters:
+        description: Active include/exclude filter patterns.
+        type: string
+      path_peerxfer:
+        description: Number of paths transferred by the peer.
+        type: integer
+      byte_peerxfer:
+        description: Number of bytes transferred by the peer.
+        type: integer
+      path_xferres:
+        description: Number of paths for which transfer was resumed.
+        type: integer
+      byte_xferres:
+        description: Number of bytes for which transfer was resumed.
+        type: integer
+      file_synced:
+        description: Number of files successfully synchronized.
+        type: integer
+      file_peerxfer:
+        description: Number of files transferred by the peer.
+        type: integer
+      file_xferres:
+        description: Number of files for which transfer was resumed.
+        type: integer
+      ascp_sids:
+        description: Last known FASP (ascp) session IDs.
+        type: string
+  counters:
+    title: sync_snap_counters_table
+    description: One row, updated live during the session with path and file counts per state.
+    type: object
+    properties:
+      syncd:
+        description: Number of paths in the `Syncd` state.
+        type: integer
+      conflict:
+        description: Number of paths in the `Conflict` state.
+        type: integer
+      potential_conflict:
+        description: Number of paths in the `Potential conflict` state.
+        type: integer
+      error:
+        description: Number of paths in the `Error` state.
+        type: integer
+      pending:
+        description: Number of paths still pending synchronization.
+        type: integer
+      unused:
+        description: Reserved (unused counter slot).
+        type: integer
+      files_syncd:
+        description: Number of files in the `Syncd` state.
+        type: integer
+      files_conflict:
+        description: Number of files in the `Conflict` state.
+        type: integer
+      files_potential_conflict:
+        description: Number of files in the `Potential conflict` state.
+        type: integer
+      files_error:
+        description: Number of files in the `Error` state.
+        type: integer
+      files_pending:
+        description: Number of files still pending synchronization.
+        type: integer
+  file_info:
+    title: sync_snapdb_table
+    description: One row per tracked file or directory. Primary source for the `file_info` command.
+    type: object
+    properties:
+      record_id:
+        description: Primary key (auto-increment).
+        type: integer
+      nodeid:
+        description: Local filesystem node ID.
+        type: integer
+      peer_rid:
+        description: Peer record ID for this entry.
+        type: integer
+      state:
+        description: |
+          Sync state of the entry.
+
+          | Value | Name              |
+          |-------|-------------------|
+          | 0     | Nil               |
+          | 1–18  | Pending (variant) |
+          | 19    | Syncd             |
+          | 20    | Error             |
+          | 21    | Conflict          |
+          | 22    | PotentialConflict |
+          | 23–24 | Pending (variant) |
+        type: integer
+      version:
+        description: Schema version field (not used).
+        type: integer
+      c_meta_type:
+        description: Current metadata — file type (file, directory, …).
+        type: integer
+      c_meta_parentrid:
+        description: Current metadata — parent record ID.
+        type: integer
+      c_meta_path:
+        description: Current metadata — file path as known to the snapshot.
+        type: string
+      c_meta_attrs:
+        description: Current metadata — file attributes.
+        type: string
+      c_meta_uid:
+        description: Current metadata — user ID (Unix).
+        type: integer
+      c_meta_gid:
+        description: Current metadata — group ID (Unix).
+        type: integer
+      c_meta_mtime:
+        description: Current metadata — modification time (epoch).
+        type: integer
+      c_meta_atime:
+        description: Current metadata — access time (epoch).
+        type: integer
+      c_meta_ctime:
+        description: Current metadata — change time (epoch).
+        type: integer
+      c_meta_xattrs:
+        description: Current metadata — extended attributes (not used).
+        type: integer
+      c_meta_xattrs_cksum:
+        description: Current metadata — checksum of extended attributes.
+        type: string
+      c_meta_ol_legal_hold:
+        description: Current metadata — object lock legal hold flag.
+        type: integer
+      c_meta_ol_mode:
+        description: Current metadata — object lock mode.
+        type: integer
+      c_meta_ol_retain_until:
+        description: Current metadata — object lock retain-until date (epoch).
+        type: integer
+      c_meta_object_metadata:
+        description: Current metadata — object metadata (JSON).
+        type: string
+      c_cont_size:
+        description: Current content — file size in bytes.
+        type: integer
+      c_cont_csum:
+        description: Current content — checksum.
+        type: string
+      c_cont_mtime:
+        description: Current content — modification time (epoch).
+        type: integer
+      c_cont_timestamp:
+        description: Current content — internal timestamp.
+        type: integer
+      p_cont_size:
+        description: Pending content — file size in bytes.
+        type: integer
+      p_cont_csum:
+        description: Pending content — checksum.
+        type: string
+      p_cont_mtime:
+        description: Pending content — modification time (epoch).
+        type: integer
+      p_cont_timestamp:
+        description: Pending content — internal timestamp.
+        type: integer
+      f_meta_type:
+        description: Filesystem metadata — file type.
+        type: integer
+      f_meta_parentrid:
+        description: Filesystem metadata — parent record ID.
+        type: integer
+      f_meta_path:
+        description: Filesystem metadata — file path as seen on disk (displayed by `file_info`).
+        type: string
+      f_meta_attrs:
+        description: Filesystem metadata — file attributes.
+        type: string
+      f_meta_uid:
+        description: Filesystem metadata — user ID (Unix).
+        type: integer
+      f_meta_gid:
+        description: Filesystem metadata — group ID (Unix).
+        type: integer
+      f_meta_mtime:
+        description: Filesystem metadata — modification time (epoch).
+        type: integer
+      f_meta_atime:
+        description: Filesystem metadata — access time (epoch).
+        type: integer
+      f_meta_ctime:
+        description: Filesystem metadata — change time (epoch).
+        type: integer
+      f_meta_xattrs:
+        description: Filesystem metadata — extended attributes (not used).
+        type: integer
+      f_meta_xattrs_cksum:
+        description: Filesystem metadata — checksum of extended attributes.
+        type: string
+      f_meta_ol_legal_hold:
+        description: Filesystem metadata — object lock legal hold flag.
+        type: integer
+      f_meta_ol_mode:
+        description: Filesystem metadata — object lock mode.
+        type: integer
+      f_meta_ol_retain_until:
+        description: Filesystem metadata — object lock retain-until date (epoch).
+        type: integer
+      f_meta_object_metadata:
+        description: Filesystem metadata — object metadata (JSON).
+        type: string
+      f_cont_size:
+        description: Filesystem content — file size in bytes.
+        type: integer
+      f_cont_csum:
+        description: Filesystem content — checksum.
+        type: string
+      f_cont_mtime:
+        description: Filesystem content — modification time (epoch).
+        type: integer
+      f_cont_timestamp:
+        description: Filesystem content — internal timestamp.
+        type: integer
+      pcr_reqno:
+        description: Peer content request sequence number.
+        type: integer
+      pp_cont_size:
+        description: Peer pending content — file size in bytes.
+        type: integer
+      pp_cont_csum:
+        description: Peer pending content — checksum.
+        type: string
+      pp_cont_mtime:
+        description: Peer pending content — modification time (epoch).
+        type: integer
+      pp_cont_timestamp:
+        description: Peer pending content — internal timestamp.
+        type: integer
+      tobe_deleted:
+        description: Set to 1 when the record is flagged for deletion (garbage collection pending).
+        type: integer
+      scn_not_visited:
+        description: Set to 1 when the scanner has not yet visited this entry in the current scan pass.
+        type: integer
+      scn_required:
+        description: Set to 1 when a rescan of this entry is required.
+        type: integer
+      scn_completed:
+        description: Set to 1 when the scan of this directory entry has completed.
+        type: integer
+      candidate_in_prog:
+        description: Set to 1 when this record is currently in the candidate transfer list.
+        type: integer
+      meta_lastreq_no:
+        description: Sequence number of the last metadata request sent.
+        type: integer
+      meta_lastresp_no:
+        description: Sequence number of the last metadata response received.
+        type: integer
+      commit_version:
+        description: Local commit version for this entry.
+        type: integer
+      peer_commit_version:
+        description: Peer commit version for this entry.
+        type: integer
+      peer_nodeid:
+        description: Peer filesystem node ID.
+        type: integer
+      recursive_mtime:
+        description: Most recent modification time across all descendants (directories only).
+        type: integer
+      message:
+        description: Error or status message associated with this entry.
+        type: string

data/lib/aspera/schema/documentation.rb

@@ -7,6 +7,15 @@ module Aspera
   module Schema
     # Generate documentation from Schema, for Transfer Spec, or async Conf spec
     class Documentation
+      # Map JSON Schema type names to user-friendly display names
+      JSON_TYPE_TO_DOC = {
+        'string'  => 'String',
+        'integer' => 'Integer',
+        'number'  => 'Number',
+        'boolean' => 'Bool',
+        'array'   => 'Array',
+        'object'  => 'Hash'
+      }.freeze
       # @param formatter [Cli::Formatter] Formatter instance with methods: markdown_text, tick, check_row
       # @param schema [Reader]
       # @param include_option [Boolean] `true`: include CLI options (switches, env vars) in descriptions
@@ -18,25 +27,27 @@ module Aspera
         @include_option = include_option
         @agent_columns = agent_columns
         @code_highlight = code_highlight
-        @columns = %i[name type description]
-        @columns.insert(-2, *Agent::Factory::ALL.values.map{ |i| i[:short]}.sort) if @agent_columns
-        # @type [Array<Hash<Symbol,String>>]
-        @rows = []
+        @columns = %w[name type description]
+        @columns.insert(-2, *Agent::Factory::ALL.values.map{ |i| i[:short].to_s}.sort) if @agent_columns
+        # Sections: each entry is {header: row_or_nil, rows: []}
+        # A flat schema produces a single section with no header.
+        @sections = [{header: nil, rows: []}]
       end
 
       def rows
-        @rows.sort_by{ |i| i[:name]}
+        @sections.flat_map do |section|
+          sorted = section[:rows].sort_by{ |i| i['name']}
+          section[:header] ? [section[:header]] + sorted : sorted
+        end
       end
 
       # @return [Array<String>]
-      def columns
-        @columns.map(&:to_s)
-      end
+      attr_reader :columns
 
-      # First row is the titles
+      # First row is the titles (for Markdown table generation)
       # @return [Array<Array<String>>]
       def table
-        [@columns.map(&:to_s)] + @rows.sort_by{ |i| i[:name]}.map{ |row| @columns.map{ |field| row[field]}}
+        [@columns] + rows.map{ |row| @columns.map{ |field| row[field]}}
       end
 
       # Generate a documentation table from a JSON schema for transfer specifications
@@ -49,37 +60,57 @@ module Aspera
       def build(schema = nil)
         code = @code_highlight ? ->(c){"`#{c}`"} : ->(c){c}
         schema ||= @schema
-        schema.each_property do |property_schema, _name, property_full_name|
+        render_title = ->(title){title.gsub(Markdown::FORMATS){@formatter.markdown_text(Regexp.last_match)}}
+        on_variant = ->(variant_reader, discriminant_property, discriminant_value) do
+          title = variant_reader.current['title'] || variant_reader.current['description']
+          header =
+            if discriminant_property && discriminant_value
+              desc = render_title.call("`#{discriminant_value}`")
+              desc += ": #{render_title.call(title)}" if title
+              @formatter.check_row({
+                'name'        => render_title.call("**#{discriminant_property}**"),
+                'type'        => code.call(JSON_TYPE_TO_DOC['string']),
+                'description' => desc
+              })
+            elsif title
+              @formatter.check_row({'name' => "**#{render_title.call(title)}**", 'type' => '&nbsp;', 'description' => '&nbsp;'})
+            end
+          @sections.push({header: header, rows: []})
+        end
+        schema.each_property(on_variant: on_variant) do |property_schema, _name, property_full_name|
           node = property_schema.current
           # Manual table
+          item_type = JSON_TYPE_TO_DOC.fetch(node['type'], node['type'])
+          item_type = "#{item_type}[#{JSON_TYPE_TO_DOC.fetch(node.dig('items', 'type'), node.dig('items', 'type'))}]" if node['type'].eql?('array') && node.dig('items', 'type').is_a?(String)
           item = {
-            name:        code.call(property_full_name),
-            type:        code.call(node['type']),
-            description: []
+            'name'        => code.call(property_full_name),
+            'type'        => code.call(item_type),
+            'description' => []
           }
           # Render Markdown formatting and split lines
-          item[:description] =
+          item['description'] =
             node['description']
               .gsub(Markdown::FORMATS){@formatter.markdown_text(Regexp.last_match)}
               .split("\n") if node.key?('description')
-          item[:description].unshift("DEPRECATED: #{node['x-deprecation']}") if node.key?('x-deprecation')
+          item['description'].unshift("DEPRECATED: #{node['x-deprecation']}") if node.key?('x-deprecation')
           # Add flags for supported agents in doc
           agents = []
           Agent::Factory::ALL.each_key do |sym|
             agents.push(sym) if node['x-agents'].nil? || node['x-agents'].include?(sym.to_s)
           end
-          Aspera.assert(agents.include?(:direct)){"#{name}: x-cli-option requires agent direct (or nil)"} if node['x-cli-option']
+          Aspera.assert(agents.include?(:direct)){"#{property_full_name}: x-cli-option requires agent direct (or nil)"} if node['x-cli-option']
           if @agent_columns
             Agent::Factory::ALL.each do |sym, names|
-              item[names[:short]] = @formatter.tick(agents.include?(sym))
+              item[names[:short].to_s] = @formatter.tick(agents.include?(sym))
             end
           else
-            item[:description].push("(#{agents.map{ |i| Agent::Factory::ALL[i][:short].to_s.upcase}.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
+            item['description'].push("(#{agents.map{ |i| Agent::Factory::ALL[i][:short].to_s.upcase}.sort.join(', ')})") unless agents.length.eql?(Agent::Factory::ALL.length)
           end
           # Only keep lines that are usable in supported agents
-          next false if agents.empty?
-          item[:description].push("Allowed values: #{node['enum'].map{ |v| @formatter.markdown_text("`#{v}`")}.join(', ')}.") if node.key?('enum')
-          item[:description].push("Default: #{code.call(node['default'])}.") if node.key?('default')
+          next if agents.empty?
+          item['description'].push("Allowed values: #{node['enum'].map{ |v| @formatter.markdown_text("`#{v}`")}.join(', ')}.") if node.key?('enum')
+          item['description'].push("Default: #{@formatter.markdown_text("`#{node['default']}`")}.") if node.key?('default')
+          item['description'].push("Example: #{@formatter.markdown_text("`#{node['example']}`")}.") if node.key?('example')
           if @include_option
             envvar_prefix = ''
             cli_option =
@@ -87,18 +118,18 @@ module Aspera
                 envvar_prefix = 'env:'
                 node['x-cli-envvar']
               elsif node['x-cli-switch']
-                node['x-cli-option']
+                false_part = node.key?('x-cli-false') ? " / #{node['x-cli-false']}" : ''
+                "#{node['x-cli-option']}#{false_part}"
               elsif node['x-cli-option']
                 arg_type = node.key?('enum') ? '{enum}' : "{#{[node['type']].flatten.join('|')}}"
-                # conversion_tag = node['x-cli-convert']
                 conversion_tag = node.key?('x-cli-convert') ? 'conversion' : nil
                 sep = node['x-cli-option'].start_with?('--') ? '=' : ' '
                 "#{node['x-cli-option']}#{sep}#{"(#{conversion_tag})" if conversion_tag}#{arg_type}"
               end
             short = node.key?('x-cli-short') ? "(#{node['x-cli-short']})" : nil
-            item[:description].push("(#{'special:' if node['x-cli-special']}#{envvar_prefix}#{@formatter.markdown_text("`#{cli_option}`")})#{short}") if cli_option
+            item['description'].push("(#{'special:' if node['x-cli-special']}#{envvar_prefix}#{@formatter.markdown_text("`#{cli_option}`")})#{short}") if cli_option
           end
-          @rows.push(@formatter.check_row(item))
+          @sections.last[:rows].push(@formatter.check_row(item))
         end
         self
       end

data/lib/aspera/schema/reader.rb

@@ -7,11 +7,34 @@ module Aspera
     class Reader
       attr_reader :current
 
+      class << self
+        # Build a synthetic Reader from an OAS `parameters` array (entries with `in: query`).
+        # Produces a JSON Schema object whose `properties` map each query param name to its schema,
+        # with the OAS-level `description` and `required` merged in.
+        # @param params [Array<Hash>] raw OAS parameter objects (may contain path/header params too)
+        # @return [Reader]
+        def from_query_params(params)
+          properties = {}
+          required_names = []
+          params.each do |param|
+            next unless param['in'] == 'query'
+            name = param['name']
+            prop = (param['schema'] || {}).dup
+            prop['description'] = param['description'] if param['description'] && !prop.key?('description')
+            properties[name] = prop
+            required_names << name if param['required']
+          end
+          synthetic = {'type' => 'object', 'properties' => properties}
+          synthetic['required'] = required_names unless required_names.empty?
+          new(synthetic)
+        end
+      end
+
       # Shortcut to access current value at path
-      # @param x [String] path element
+      # @param key [String] path element
       # @return [Hash, Array, String, Integer] current value at path
-      def [](x)
-        @current[x]
+      def [](key)
+        @current[key]
       end
 
       # Find sub path relative to current
@@ -24,13 +47,19 @@ module Aspera
           Aspera.assert_type(current, Hash){'schema'}
           if current.key?('$ref')
             ref = current['$ref']
-            Aspera.assert(ref.start_with?('#/'))
+            Aspera.assert(ref.start_with?('#/')){"schema $ref must start with '#/': #{ref}"}
             current = @root.dig(*ref[2..].split('/'))
           end
         end
         Reader.new(@root, current)
       end
 
+      # Resolve a $ref string to a Reader
+      def resolve_ref(ref)
+        Aspera.assert(ref.start_with?('#/')){"schema $ref must start with '#/': #{ref}"}
+        Reader.new(@root, @root.dig(*ref[2..].split('/')))
+      end
+
       # Read schema from file or from cache
       # @param root [Hash] root schema
       # @param current [Hash, nil] current position in
@@ -40,15 +69,35 @@ module Aspera
         @current = current || root
       end
 
-      # Recursively traverse schema properties with a block
-      # Handles nested objects and arrays automatically
-      # @param prefix [String] Prefix for property names (e.g., 'parent.child.')
+      # Recursively traverse schema properties with a block.
+      # If the current node has `oneOf`, each variant is traversed in turn and
+      # `on_variant` is called (if given) before each variant's properties.
+      # @param prefix     [String] Prefix for property names (e.g., 'parent.child.')
+      # @param on_variant [Proc, nil] Called with the variant Reader before its properties
       # @yield [property_schema, name, full_name] Yields property info to block
-      # @yieldparam property_schema [Reader] Schema reader for this property (use .current to get node hash)
+      # @yieldparam property_schema [Reader] Schema reader for this property
       # @yieldparam name [String] Property name
       # @yieldparam full_name [String] Full property name with prefix
       # @return [nil]
-      def each_property(prefix = '', &block)
+      def each_property(prefix = '', on_variant: nil, &block)
+        if @current.key?('oneOf')
+          # Build reverse map: $ref -> discriminant value, from discriminator.mapping if present
+          discriminant_by_ref = {}
+          if @current.dig('discriminator', 'mapping').is_a?(Hash)
+            @current['discriminator']['mapping'].each do |value, ref|
+              discriminant_by_ref[ref] = value
+            end
+          end
+          discriminant_property = @current.dig('discriminator', 'propertyName')
+          @current['oneOf'].each do |variant_node|
+            ref = variant_node['$ref']
+            variant_reader = ref ? resolve_ref(ref) : Reader.new(@root, variant_node)
+            discriminant_value = ref ? discriminant_by_ref[ref] : nil
+            on_variant&.call(variant_reader, discriminant_property, discriminant_value)
+            variant_reader.each_property(prefix, on_variant: on_variant, &block)
+          end
+          return
+        end
         properties = dig('properties')
         properties.current.each_key do |name|
           property_full_name = "#{prefix}#{name}"
@@ -61,11 +110,11 @@ module Aspera
           # Recursively process nested structures
           case node['type']
           when 'object'
-            property_schema.each_property("#{property_full_name}.", &block) if node['properties']
+            property_schema.each_property("#{property_full_name}.", on_variant: on_variant, &block) if node['properties']
           when 'array'
             if node['items']
               array_item_schema = property_schema.dig('items')
-              array_item_schema.each_property("#{property_full_name}[].", &block) if array_item_schema.current['properties']
+              array_item_schema.each_property("#{property_full_name}[].", on_variant: on_variant, &block) if array_item_schema.current['properties']
             end
           end
         end