aspera-cli 4.18.1 → 4.20.0

data/lib/aspera/api/httpgw.rb

@@ -45,7 +45,7 @@ module Aspera
         else
           @shared_info[:count][:sent_general] += 1
         end
-        Log.log.debug do
+        Log.log.trace1 do
           log_data = payload.dup
           log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
           "#{LOG_WS_SEND}json: #{msg_type}: #{JSON.generate(log_data)}"
@@ -55,7 +55,7 @@ module Aspera
 
       # send data on http gw web socket
       def ws_send(ws_type:, data:)
-        Log.log.debug{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
+        Log.log.trace1{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
         @shared_info[:count][:sent_general] += 1 if ws_type.eql?(:binary)
         frame_generator = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: ws_type, version: @ws_handshake.version)
         @ws_io.write(frame_generator.to_s)
@@ -68,13 +68,13 @@ module Aspera
                 (((@shared_info[:count][:sent_general] - @shared_info[:count][:received_general]) > 1) ||
                   ((@shared_info[:count][:received_v2_delimiter] - @shared_info[:count][:sent_v2_delimiter]) > 1))
               if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
-                Log.log.debug{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
+                Log.log.trace1{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
               end
             end
           end
         end
         raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
-        Log.log.debug{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
+        Log.log.trace2{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
       end
 
       # message processing for read thread
@@ -114,12 +114,12 @@ module Aspera
             # ready byte by byte until frame is ready
             # blocking read
             byte = @ws_io.read(1)
-            Log.log.trace1{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
+            Log.log.trace2{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
             frame_parser << byte
             frame_ok = frame_parser.next
             next if frame_ok.nil?
             process_received_message(frame_ok.data.to_s)
-            Log.log.debug{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
+            Log.log.trace2{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
           rescue => e
             Log.log.debug{"#{LOG_WS_RECV}Exception: #{e}"}
             @shared_info[:mutex].synchronize do
@@ -138,38 +138,16 @@ module Aspera
       def upload(transfer_spec)
         # identify this session uniquely
         session_id = SecureRandom.uuid
-        @notify_cb&.call(session_id: nil, type: :pre_start, info: 'starting')
-        # total size of all files
-        total_bytes_to_transfer = 0
-        # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
-        files_to_read = []
-        # get source root or nil
-        source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
-        # source root is ignored by GW, used only here
-        transfer_spec.delete('source_root')
-        # compute total size of files to upload (for progress)
-        # modify transfer spec to be suitable for GW
-        transfer_spec['paths'].each do |item|
-          # save actual file location to be able read contents later
-          file_to_add = Transfer::FauxFile.open(item['source'])
-          if file_to_add
-            item['source'] = file_to_add.path
-            item['file_size'] = file_to_add.size
-          else
-            file_to_add = item['source']
-            # add source root if needed
-            file_to_add = File.join(source_root, file_to_add) unless source_root.nil?
-            # GW expects a simple file name in 'source' but if user wants to change the name, we take it
-            item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
-            item['file_size'] = File.size(file_to_add)
-          end
-          # save so that we can actually read the file later
-          files_to_read.push(file_to_add)
-          total_bytes_to_transfer += item['file_size']
-        end
+        @notify_cb&.call(:pre_start, session_id: nil, info: 'starting')
+        # process files to send, modify `paths` in transfer_spec
+        files_to_send = process_upload_list(transfer_spec)
+        # total size of all files is last element
+        total_bytes_to_transfer = files_to_send.pop
+        Log.log.trace1{Log.dump(:modified_tspec, transfer_spec)}
+        Log.log.trace1{Log.dump(:files_to_send, files_to_send)}
         # TODO: check that this is available in endpoints: @api_info['endpoints']
         upload_url = File.join(@gw_root_url, @upload_version, 'upload')
-        @notify_cb&.call(session_id: nil, type: :pre_start, info: 'connecting wss')
+        @notify_cb&.call(:pre_start, session_id: nil, info: 'connecting wss')
         # open web socket to end point (equivalent to Net::HTTP.start)
         http_session = Rest.start_http_session(upload_url)
         # get the underlying socket i/o
@@ -180,11 +158,6 @@ module Aspera
         @ws_handshake << @ws_io.readuntil("\r\n\r\n")
         Aspera.assert(@ws_handshake.finished?){'Error in websocket handshake'}
         Log.log.debug{"#{LOG_WS_SEND}handshake success"}
-        # start read thread after handshake
-        @ws_read_thread = Thread.new {process_read_thread}
-        @notify_cb&.call(session_id: session_id, type: :session_start)
-        @notify_cb&.call(session_id: session_id, type: :session_size, info: total_bytes_to_transfer)
-        sleep(1)
         # data shared between main thread and read thread
         @shared_info = {
           read_exception: nil, # error message if any in callback
@@ -197,34 +170,34 @@ module Aspera
           mutex:          Mutex.new,
           cond_var:       ConditionVariable.new
         }
+        # start read thread after handshake
+        @ws_read_thread = Thread.new {process_read_thread}
+        @notify_cb&.call(:session_start, session_id: session_id)
+        @notify_cb&.call(:session_size, session_id: session_id, info: total_bytes_to_transfer)
+        sleep(1)
         # notify progress bar
-        @notify_cb&.call(type: :session_size, session_id: session_id, info: total_bytes_to_transfer)
+        @notify_cb&.call(:session_size, session_id: session_id, info: total_bytes_to_transfer)
         # first step send transfer spec
-        Log.log.debug{Log.dump(:ws_spec, transfer_spec)}
         ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
         # current file index
         file_index = 0
         # aggregate size sent
         session_sent_bytes = 0
         # process each file
-        transfer_spec['paths'].each do |item|
+        files_to_send.each do |file_to_send|
+          last_slice = (file_to_send[:size] - 1) / @upload_chunk_size
           slice_info = {
-            name:       nil,
+            name:         file_to_send[:name],
             # TODO: get mime type?
-            type:       'application/octet-stream',
-            size:       item['file_size'],
-            slice:      0, # current slice index
+            type:         'application/octet-stream',
+            size:         file_to_send[:size],
+            slice:        0, # current slice index
             # index of last slice (i.e number of slices - 1)
-            last_slice: (item['file_size'] - 1) / @upload_chunk_size,
-            fileIndex:  file_index
+            total_slices: last_slice + 1,
+            fileIndex:    file_index
           }
-          file = files_to_read[file_index]
-          if file.is_a?(Transfer::FauxFile)
-            slice_info[:name] = file.path
-          else
-            file = File.open(file)
-            slice_info[:name] = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
-          end
+          file = file_to_send[:file]
+          file = File.open(file) unless file.is_a?(Transfer::FauxFile)
           begin
             until file.eof?
               slice_bin_data = file.read(@upload_chunk_size)
@@ -238,9 +211,9 @@ module Aspera
                   # send once, before data, at beginning
                   ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(0)
                   ws_send(ws_type: :binary, data: slice_bin_data)
-                  Log.log.debug{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{slice_info[:last_slice]}"}
+                  Log.log.trace1{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{last_slice}"}
                   # send once, after data, at end
-                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(slice_info[:last_slice])
+                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(last_slice)
                 end
               rescue Errno::EPIPE => e
                 raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
@@ -250,7 +223,7 @@ module Aspera
                 raise e
               end
               session_sent_bytes += slice_bin_data.length
-              @notify_cb&.call(type: :transfer, session_id: session_id, info: session_sent_bytes)
+              @notify_cb&.call(:transfer, session_id: session_id, info: session_sent_bytes)
               slice_info[:slice] += 1
             end
           ensure
@@ -259,8 +232,8 @@ module Aspera
           file_index += 1
         end
         # throttling may have skipped last one
-        @notify_cb&.call(type: :transfer, session_id: session_id, info: session_sent_bytes)
-        @notify_cb&.call(type: :end, session_id: session_id)
+        @notify_cb&.call(:transfer, session_id: session_id, info: session_sent_bytes)
+        @notify_cb&.call(:end, session_id: session_id)
         ws_send(ws_type: :close, data: nil)
         Log.log.debug("Finished upload, waiting for end of #{THR_RECV} thread.")
         @ws_read_thread.join
@@ -283,7 +256,7 @@ module Aspera
           end
           transfer_spec['download_name'] = download_name
         end
-        creation = create('download', {'transfer_spec' => transfer_spec})[:data]
+        creation = create('download', {'transfer_spec' => transfer_spec})
         transfer_uuid = creation['url'].split('/').last
         file_name =
           if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
@@ -301,6 +274,11 @@ module Aspera
         return @api_info
       end
 
+      # @return the base url of the gateway
+      def base_url
+        return @gw_root_url
+      end
+
       # @param url [String] URL of the HTTP Gateway, without version
       def initialize(
         url:,
@@ -326,7 +304,7 @@ module Aspera
         @synchronous = synchronous
         @notify_cb = notify_cb
         # get API info
-        @api_info = read('info')[:data].freeze
+        @api_info = read('info').freeze
         Log.log.debug{Log.dump(:api_info, @api_info)}
         # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
         # is the latest supported? else revert to old api
@@ -337,6 +315,68 @@ module Aspera
           end
         end
       end
+
+      private
+
+      # compute total size of files to upload (for progress)
+      # modify transfer spec to be suitable for HTTPGW
+      # @param transfer_spec [Hash] transfer specification
+      # @return [Array] info on files to send
+      def process_upload_list(transfer_spec)
+        total_bytes_to_transfer = 0
+        source_prefix = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] + '/' : ''
+        files_to_send = []
+        transfer_spec['paths'].each do |one_path|
+          source_path = source_prefix + one_path['source']
+          faux_file = Transfer::FauxFile.create(source_path)
+          if faux_file
+            total_bytes_to_transfer += faux_file.size
+            files_to_send.push({
+              file: faux_file,
+              name: faux_file.path,
+              size: faux_file.size
+            })
+          elsif File.file?(source_path)
+            # regular file
+            file_size = File.size(source_path)
+            total_bytes_to_transfer += file_size
+            files_to_send.push({
+              file: source_path,
+              # GW expects a simple file name in 'source' but if user wants to change the name, we take it
+              name: File.basename(one_path['destination'].nil? ? source_path : one_path['destination']),
+              size: file_size
+            })
+          elsif File.directory?(source_path)
+            folders_to_process = [source_path]
+            until folders_to_process.empty?
+              folder = folders_to_process.shift
+              # read all entries
+              Dir.entries(folder).each do |entry|
+                next if entry.eql?('.') || entry.eql?('..')
+                entry_path = File.join(folder, entry)
+                if File.directory?(entry_path)
+                  folders_to_process.push(entry_path)
+                elsif File.file?(entry_path)
+                  file_size = File.size(entry_path)
+                  total_bytes_to_transfer += file_size
+                  files_to_send.push({
+                    file: entry_path,
+                    name: entry_path,
+                    size: file_size
+                  })
+                else
+                  Log.log.warn{"Ignoring non file/directory: #{entry_path}"}
+                end
+              end
+            end
+          else
+            raise "File not found: #{source_path}"
+          end
+        end
+        transfer_spec['paths'] = files_to_send.map{|i|{'source' => i[:name]}}
+        files_to_send.push(total_bytes_to_transfer)
+        return files_to_send
+      end
     end
   end
 end

data/lib/aspera/api/node.rb

@@ -33,6 +33,8 @@ module Aspera
       ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
       HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
       HEADER_X_TOTAL_COUNT = 'X-Total-Count'
+      HEADER_X_CACHE_CONTROL = 'X-Aspera-Cache-Control'
+      HEADER_X_NEXT_ITER_TOKEN = 'X-Aspera-Next-Iteration-Token'
       SCOPE_USER = 'user:all'
       SCOPE_ADMIN = 'admin:all'
       PATH_SEPARATOR = '/'
@@ -42,9 +44,17 @@ module Aspera
 
       # class instance variable, access with accessors on class
       @use_standard_ports = true
+      @use_node_cache = true
 
       class << self
         attr_accessor :use_standard_ports
+        attr_accessor :use_node_cache
+
+        def cache_control_headers
+          h = {'Accept' => 'application/json'}
+          h[HEADER_X_CACHE_CONTROL] = 'no-cache' unless use_node_cache
+          h
+        end
 
         # For access keys: provide expression to match entry in folder
         def file_matcher(match_expression)
@@ -146,6 +156,11 @@ module Aspera
         end
       end
 
+      # Call node API, possibly adding cache control header, as globally specified
+      def read_with_cache(subpath, query=nil)
+        return call(operation: 'GET', subpath: subpath, headers: self.class.cache_control_headers, query: query)[:data]
+      end
+
       # update transfer spec with special additional tags
       def add_tspec_info(tspec)
         tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
@@ -171,7 +186,7 @@ module Aspera
       def entry_has_link_information(entry)
         # if target information is missing in folder, try to get it on entry
         if entry['target_node_id'].nil? || entry['target_id'].nil?
-          link_entry = read("files/#{entry['id']}")[:data]
+          link_entry = read("files/#{entry['id']}")
           entry['target_node_id'] = link_entry['target_node_id']
           entry['target_id'] = link_entry['target_id']
         end
@@ -200,13 +215,19 @@ module Aspera
           # get folder content
           folder_contents =
             begin
-              read("files/#{current_item[:id]}/files")[:data]
+              read("files/#{current_item[:id]}/files")
             rescue StandardError => e
               Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
               []
             end
           Log.log.debug{Log.dump(:folder_contents, folder_contents)}
           folder_contents.each do |entry|
+            if entry.key?('error')
+              if entry['error'].is_a?(Hash) && entry['error'].key?('user_message')
+                Log.log.error(entry['error']['user_message'])
+              end
+              next
+            end
             relative_path = File.join(current_item[:path], entry['name'])
             Log.log.debug{"process_folder_tree: checking #{relative_path}"}
             # call block, continue only if method returns true
@@ -228,16 +249,16 @@ module Aspera
         end
       end
 
-      # Navigate the path from given file id
+      # Navigate the path from given file id on current node, and return the node and file id of target.
+      # If the path ends with a "/" or process_last_link is true then if the last item in path is a link, it is followed.
       # @param top_file_id [String] id initial file id
-      # @param path [String]  file path
+      # @param path [String] file or folder path (end with "/" is like setting process_last_link)
+      # @param process_last_link [Boolean] if true, follow the last link
       # @return [Hash] {.api,.file_id}
-      def resolve_api_fid(top_file_id, path)
+      def resolve_api_fid(top_file_id, path, process_last_link=false)
         Aspera.assert_type(top_file_id, String)
         Aspera.assert_type(path, String)
-        # if last element is a link and followed by "/", we list the content of that folder, else we return the link
-        process_last_link = path.end_with?(PATH_SEPARATOR)
-        # keep only non-empty elements
+        process_last_link ||= path.end_with?(PATH_SEPARATOR)
         path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
         return {api: self, file_id: top_file_id} if path_elements.empty?
         resolve_state = {path: path_elements, result: nil, process_last_link: process_last_link}
@@ -265,7 +286,7 @@ module Aspera
           full_spec = create(
             'files/download_setup',
             {transfer_requests: [{transfer_request: {paths: [{source: '/'}]}}]}
-          )[:data]['transfer_specs'].first['transfer_spec']
+          )['transfer_specs'].first['transfer_spec']
           # set available fields
           @std_t_spec_cache = Transfer::Spec::TRANSPORT_FIELDS.each_with_object({}) do |i, h|
             h[i] = full_spec[i] if full_spec.key?(i)
@@ -317,13 +338,13 @@ module Aspera
           if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
             transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
           end
-          info = read('info')[:data]
+          info = read('info')
           # get the transfer user from info on access key
           transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
           # get settings from name.value array to hash key.value
           settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
           # check WSS ports
-          %w[wss_enabled wss_port].each do |i|
+          Transfer::Spec::WSS_FIELDS.each do |i|
             transfer_spec[i] = settings[i] if settings.key?(i)
           end if settings.is_a?(Hash)
         else
@@ -380,7 +401,7 @@ module Aspera
         return true
       end
 
-      def process_find_files(entry, _path, state)
+      def process_find_files(entry, path, state)
         state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
         # test all files deeply
         return true

data/lib/aspera/ascmd.rb

@@ -22,7 +22,54 @@ module Aspera
       mv:     2,
       rm:     1
     }.freeze
-    private_constant :OPS_ARGS
+
+    # protocol is based on Type-Length-Value
+    # type start at one, but array index start at zero
+    ENUM_START = 1
+
+    # description of result structures (see ascmdtypes.h).
+    # Base types are big endian
+    # key = name of type
+    # index in array `fields` is the type (minus ENUM_START)
+    # decoding always start at `result`
+    # some fields have special handling indicated by `special`
+    # field_list, list_tlv_list, list_tlv_restart are composed with a list of TLV
+    TYPES_DESCR = {
+      result: {decode: :field_list,
+               fields: [{name: :file, is_a: :stat}, {name: :dir, is_a: :stat, special: :list_tlv_list}, {name: :size, is_a: :size}, {name: :error, is_a: :error},
+                        {name: :info, is_a: :info}, {name: :success, is_a: nil, special: :return_true}, {name: :exit, is_a: nil},
+                        {name: :df, is_a: :mnt, special: :list_tlv_restart}, {name: :md5sum, is_a: :md5sum}]},
+      stat:   {decode: :field_list,
+               fields: [{name: :name, is_a: :zstr}, {name: :size, is_a: :int64}, {name: :mode, is_a: :int32, check: nil}, {name: :zmode, is_a: :zstr},
+                        {name: :uid, is_a: :int32, check: nil}, {name: :zuid, is_a: :zstr}, {name: :gid, is_a: :int32, check: nil}, {name: :zgid, is_a: :zstr},
+                        {name: :ctime, is_a: :epoch}, {name: :zctime, is_a: :zstr}, {name: :mtime, is_a: :epoch}, {name: :zmtime, is_a: :zstr},
+                        {name: :atime, is_a: :epoch}, {name: :zatime, is_a: :zstr}, {name: :symlink, is_a: :zstr}, {name: :errno, is_a: :int32},
+                        {name: :errstr, is_a: :zstr}]},
+      info:   {decode: :field_list,
+               fields: [{name: :platform, is_a: :zstr}, {name: :version, is_a: :zstr}, {name: :lang, is_a: :zstr}, {name: :territory, is_a: :zstr},
+                        {name: :codeset, is_a: :zstr}, {name: :lc_ctype, is_a: :zstr}, {name: :lc_numeric, is_a: :zstr}, {name: :lc_time, is_a: :zstr},
+                        {name: :lc_all, is_a: :zstr}, {name: :dev, is_a: :zstr, special: :list_multiple}, {name: :browse_caps, is_a: :zstr},
+                        {name: :protocol, is_a: :zstr}]},
+      size:   {decode: :field_list,
+               fields: [{name: :size, is_a: :int64}, {name: :fcount, is_a: :int32}, {name: :dcount, is_a: :int32}, {name: :failed_fcount, is_a: :int32},
+                        {name: :failed_dcount, is_a: :int32}]},
+      error:  {decode: :field_list,
+               fields: [{name: :errno, is_a: :int32}, {name: :errstr, is_a: :zstr}]},
+      mnt:    {decode: :field_list,
+               fields: [{name: :fs, is_a: :zstr}, {name: :dir, is_a: :zstr}, {name: :is_a, is_a: :zstr}, {name: :total, is_a: :int64},
+                        {name: :used, is_a: :int64}, {name: :free, is_a: :int64}, {name: :fcount, is_a: :int64}, {name: :errno, is_a: :int32},
+                        {name: :errstr, is_a: :zstr}]},
+      md5sum: {decode: :field_list, fields: [{name: :md5sum, is_a: :zstr}]},
+      int8:   {decode: :base, unpack: 'C', size: 1},
+      int32:  {decode: :base, unpack: 'L>', size: 4},
+      int64:  {decode: :base, unpack: 'Q>', size: 8},
+      epoch:  {decode: :base, unpack: 'Q>', size: 8},
+      zstr:   {decode: :base, unpack: 'Z*'},
+      blist:  {decode: :buffer_list}
+    }.freeze
+
+    private_constant :TYPES_DESCR, :ENUM_START, :OPS_ARGS
+
     # list of supported actions
     OPERATIONS = OPS_ARGS.keys.freeze
 
@@ -55,6 +102,7 @@ module Aspera
       arg_batches.each do |args|
         command = [main_command]
         # enclose arguments in double quotes, protect backslash and double quotes
+        # ascmd uses space as token separator, and optional quotes ('") or \ to escape
         args.each do |v|
           command.push(%Q{"#{v.gsub(/["\\]/){|s|"\\#{s}"}}"})
         end
@@ -106,47 +154,6 @@ module Aspera
       def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments&.join(',')}"; end
     end
 
-    # description of result structures (see ascmdtypes.h). Base types are big endian
-    # key = name of type
-    TYPES_DESCR = {
-      result: {decode: :field_list,
-               fields: [{name: :file, is_a: :stat}, {name: :dir, is_a: :stat, special: :sub_struct}, {name: :size, is_a: :size}, {name: :error, is_a: :error},
-                        {name: :info, is_a: :info}, {name: :success, is_a: nil, special: :return_true}, {name: :exit, is_a: nil},
-                        {name: :df, is_a: :mnt, special: :restart_on_first}, {name: :md5sum, is_a: :md5sum}]},
-      stat:   {decode: :field_list,
-               fields: [{name: :name, is_a: :zstr}, {name: :size, is_a: :int64}, {name: :mode, is_a: :int32, check: nil}, {name: :zmode, is_a: :zstr},
-                        {name: :uid, is_a: :int32, check: nil}, {name: :zuid, is_a: :zstr}, {name: :gid, is_a: :int32, check: nil}, {name: :zgid, is_a: :zstr},
-                        {name: :ctime, is_a: :epoch}, {name: :zctime, is_a: :zstr}, {name: :mtime, is_a: :epoch}, {name: :zmtime, is_a: :zstr},
-                        {name: :atime, is_a: :epoch}, {name: :zatime, is_a: :zstr}, {name: :symlink, is_a: :zstr}, {name: :errno, is_a: :int32},
-                        {name: :errstr, is_a: :zstr}]},
-      info:   {decode: :field_list,
-               fields: [{name: :platform, is_a: :zstr}, {name: :version, is_a: :zstr}, {name: :lang, is_a: :zstr}, {name: :territory, is_a: :zstr},
-                        {name: :codeset, is_a: :zstr}, {name: :lc_ctype, is_a: :zstr}, {name: :lc_numeric, is_a: :zstr}, {name: :lc_time, is_a: :zstr},
-                        {name: :lc_all, is_a: :zstr}, {name: :dev, is_a: :zstr, special: :multiple}, {name: :browse_caps, is_a: :zstr},
-                        {name: :protocol, is_a: :zstr}]},
-      size:   {decode: :field_list,
-               fields: [{name: :size, is_a: :int64}, {name: :fcount, is_a: :int32}, {name: :dcount, is_a: :int32}, {name: :failed_fcount, is_a: :int32},
-                        {name: :failed_dcount, is_a: :int32}]},
-      error:  {decode: :field_list,
-               fields: [{name: :errno, is_a: :int32}, {name: :errstr, is_a: :zstr}]},
-      mnt:    {decode: :field_list,
-               fields: [{name: :fs, is_a: :zstr}, {name: :dir, is_a: :zstr}, {name: :is_a, is_a: :zstr}, {name: :total, is_a: :int64},
-                        {name: :used, is_a: :int64}, {name: :free, is_a: :int64}, {name: :fcount, is_a: :int64}, {name: :errno, is_a: :int32},
-                        {name: :errstr, is_a: :zstr}]},
-      md5sum: {decode: :field_list, fields: [{name: :md5sum, is_a: :zstr}]},
-      int8:   {decode: :base, unpack: 'C', size: 1},
-      int32:  {decode: :base, unpack: 'L>', size: 4},
-      int64:  {decode: :base, unpack: 'Q>', size: 8},
-      epoch:  {decode: :base, unpack: 'Q>', size: 8},
-      zstr:   {decode: :base, unpack: 'Z*'},
-      blist:  {decode: :buffer_list}
-    }.freeze
-
-    # protocol enum start at one, but array index start at zero
-    ENUM_START = 1
-
-    private_constant :TYPES_DESCR, :ENUM_START
-
     class << self
       # get description of structure's field, @param struct_name, @param typed_buffer provides field name
       def field_description(struct_name, typed_buffer)
@@ -175,6 +182,7 @@ module Aspera
           Log.log.trace1{"#{'   .' * indent_level}-> base:#{byte_array} -> #{result}"}
           result = Time.at(result) if type_name.eql?(:epoch)
         when :buffer_list
+          # return a list of type_buffer
           result = []
           until buffer.empty?
             btype = parse(buffer, :int8, indent_level)
@@ -193,16 +201,16 @@ module Aspera
             field_info = field_description(type_name, typed_buffer)
             Log.log.trace1{"#{'   .' * indent_level}+ field(special=#{field_info[:special]})=#{field_info[:name]}".green}
             case field_info[:special]
-            when nil
+            when nil # normal case
               result[field_info[:name]] = parse(typed_buffer[:buffer], field_info[:is_a], indent_level)
-            when :return_true
+            when :return_true # nothing to parse, just return true
               result[field_info[:name]] = true
-            when :sub_struct
-              result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{|r|parse(r[:buffer], field_info[:is_a], indent_level)}
-            when :multiple
+            when :list_multiple # field appears multiple times, and is an array of values (base type)
               result[field_info[:name]] ||= []
               result[field_info[:name]].push(parse(typed_buffer[:buffer], field_info[:is_a], indent_level))
-            when :restart_on_first
+            when :list_tlv_list # field is an array of values in a list of buffers
+              result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{|r|parse(r[:buffer], field_info[:is_a], indent_level)}
+            when :list_tlv_restart # field is an array of values, but a new value is started on index 1
               fl = result[field_info[:name]] = []
               parse(typed_buffer[:buffer], :blist, indent_level).map do |tb|
                 fl.push({}) if tb[:btype].eql?(ENUM_START)