aspera-cli 4.19.0 → 4.21.1

data/lib/aspera/api/node.rb

@@ -16,8 +16,6 @@ module Aspera
     class Node < Aspera::Rest
       SCOPE_SEPARATOR = ':'
       SCOPE_NODE_PREFIX = 'node.'
-      # prefix for ruby code for filter (deprecated)
-      MATCH_EXEC_PREFIX = 'exec:'
       MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
       SIGNATURE_DELIMITER = '==SIGNATURE=='
       BEARER_TOKEN_VALIDITY_DEFAULT = 86400
@@ -25,7 +23,7 @@ module Aspera
       REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
       # methods of @app_info[:api]
       REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
-      private_constant :SCOPE_SEPARATOR, :SCOPE_NODE_PREFIX, :MATCH_EXEC_PREFIX, :MATCH_TYPES,
+      private_constant :SCOPE_SEPARATOR, :SCOPE_NODE_PREFIX, :MATCH_TYPES,
         :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT,
         :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
 
@@ -33,6 +31,8 @@ module Aspera
       ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
       HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
       HEADER_X_TOTAL_COUNT = 'X-Total-Count'
+      HEADER_X_CACHE_CONTROL = 'X-Aspera-Cache-Control'
+      HEADER_X_NEXT_ITER_TOKEN = 'X-Aspera-Next-Iteration-Token'
       SCOPE_USER = 'user:all'
       SCOPE_ADMIN = 'admin:all'
       PATH_SEPARATOR = '/'
@@ -42,22 +42,23 @@ module Aspera
 
       # class instance variable, access with accessors on class
       @use_standard_ports = true
+      @use_node_cache = true
 
       class << self
+        # set to false to read transfer parameters from download_setup
         attr_accessor :use_standard_ports
+        # set to false to bypass cache in redis
+        attr_accessor :use_node_cache
 
         # For access keys: provide expression to match entry in folder
+        # @param match_expression one of supported types
+        # @return lambda function
         def file_matcher(match_expression)
           case match_expression
           when Proc then return match_expression
           when Regexp then return ->(f){f['name'].match?(match_expression)}
           when String
-            if match_expression.start_with?(MATCH_EXEC_PREFIX)
-              code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
-              Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
-              return Environment.secure_eval(code, __FILE__, __LINE__)
-            end
-            return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
+            return ->(f){File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
           when NilClass then return ->(_){true}
           else Aspera.error_unexpected_value(match_expression.class.name, exception_class: Cli::BadArgument)
           end
@@ -146,6 +147,17 @@ module Aspera
         end
       end
 
+      # Call node API, possibly adding cache control header, as globally specified
+      def read_with_cache(subpath, query=nil)
+        headers = {'Accept' => 'application/json'}
+        headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless self.class.use_node_cache
+        return call(
+          operation: 'GET',
+          subpath:   subpath,
+          headers:   headers,
+          query:     query)[:data]
+      end
+
       # update transfer spec with special additional tags
       def add_tspec_info(tspec)
         tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
@@ -171,7 +183,7 @@ module Aspera
       def entry_has_link_information(entry)
         # if target information is missing in folder, try to get it on entry
         if entry['target_node_id'].nil? || entry['target_id'].nil?
-          link_entry = read("files/#{entry['id']}")[:data]
+          link_entry = read("files/#{entry['id']}")
           entry['target_node_id'] = link_entry['target_node_id']
           entry['target_id'] = link_entry['target_id']
         end
@@ -183,11 +195,11 @@ module Aspera
       # Recursively browse in a folder (with non-recursive method)
       # sub folders are processed if the processing method returns true
       # links are processed on the respective node
+      # @param method_sym [Symbol] processing method, arguments: entry, path, state
       # @param state [Object] state object sent to processing method
       # @param top_file_id [String] file id to start at (default = access key root file id)
       # @param top_file_path [String] path of top folder (default = /)
-      # @param block [Proc] processing method, arguments: entry, path, state
-      def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/')
+      def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/', query: nil)
         Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
         Log.log.debug{"process_folder_tree: node=#{@app_info ? @app_info[:node_info]['id'] : 'nil'}, file id=#{top_file_id},  path=#{top_file_path}"}
         # start at top folder
@@ -200,44 +212,51 @@ module Aspera
           # get folder content
           folder_contents =
             begin
-              read("files/#{current_item[:id]}/files")[:data]
+              # TODO: use header
+              read_with_cache("files/#{current_item[:id]}/files")
             rescue StandardError => e
               Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
               []
             end
           Log.log.debug{Log.dump(:folder_contents, folder_contents)}
           folder_contents.each do |entry|
-            relative_path = File.join(current_item[:path], entry['name'])
-            Log.log.debug{"process_folder_tree: checking #{relative_path}"}
+            if entry.key?('error')
+              if entry['error'].is_a?(Hash) && entry['error'].key?('user_message')
+                Log.log.error(entry['error']['user_message'])
+              end
+              next
+            end
+            current_path = File.join(current_item[:path], entry['name'])
+            Log.log.debug{"process_folder_tree: checking #{current_path}"}
             # call block, continue only if method returns true
-            next unless send(method_sym, entry, relative_path, state)
+            next unless send(method_sym, entry, current_path, state)
             # entry type is file, folder or link
             case entry['type']
             when 'folder'
-              folders_to_explore.push({id: entry['id'], path: relative_path})
+              folders_to_explore.push({id: entry['id'], path: current_path})
             when 'link'
               if entry_has_link_information(entry)
                 node_id_to_node(entry['target_node_id'])&.process_folder_tree(
                   method_sym:    method_sym,
                   state:         state,
                   top_file_id:   entry['target_id'],
-                  top_file_path: relative_path)
+                  top_file_path: current_path)
               end
             end
           end
         end
       end
 
-      # Navigate the path from given file id
+      # Navigate the path from given file id on current node, and return the node and file id of target.
+      # If the path ends with a "/" or process_last_link is true then if the last item in path is a link, it is followed.
       # @param top_file_id [String] id initial file id
-      # @param path [String]  file path
+      # @param path [String] file or folder path (end with "/" is like setting process_last_link)
+      # @param process_last_link [Boolean] if true, follow the last link
       # @return [Hash] {.api,.file_id}
-      def resolve_api_fid(top_file_id, path)
+      def resolve_api_fid(top_file_id, path, process_last_link=false)
         Aspera.assert_type(top_file_id, String)
         Aspera.assert_type(path, String)
-        # if last element is a link and followed by "/", we list the content of that folder, else we return the link
-        process_last_link = path.end_with?(PATH_SEPARATOR)
-        # keep only non-empty elements
+        process_last_link ||= path.end_with?(PATH_SEPARATOR)
         path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
         return {api: self, file_id: top_file_id} if path_elements.empty?
         resolve_state = {path: path_elements, result: nil, process_last_link: process_last_link}
@@ -247,13 +266,19 @@ module Aspera
         return resolve_state[:result]
       end
 
-      def find_files(top_file_id, test_block)
+      def find_files(top_file_id, test_lambda)
         Log.log.debug{"find_files: file id=#{top_file_id}"}
-        find_state = {found: [], test_block: test_block}
+        find_state = {found: [], test_lambda: test_lambda}
         process_folder_tree(method_sym: :process_find_files, state: find_state, top_file_id: top_file_id)
         return find_state[:found]
       end
 
+      def list_files(top_file_id)
+        find_state = {found: []}
+        process_folder_tree(method_sym: :process_list_files, state: find_state, top_file_id: top_file_id)
+        return find_state[:found]
+      end
+
       def refreshed_transfer_token
         return oauth.token(refresh: true)
       end
@@ -265,7 +290,7 @@ module Aspera
           full_spec = create(
             'files/download_setup',
             {transfer_requests: [{transfer_request: {paths: [{source: '/'}]}}]}
-          )[:data]['transfer_specs'].first['transfer_spec']
+          )['transfer_specs'].first['transfer_spec']
           # set available fields
           @std_t_spec_cache = Transfer::Spec::TRANSPORT_FIELDS.each_with_object({}) do |i, h|
             h[i] = full_spec[i] if full_spec.key?(i)
@@ -275,6 +300,9 @@ module Aspera
       end
 
       # Create transfer spec for gen4
+      # @param file_id destination or source folder (id)
+      # @param direction one of Transfer::Spec::DIRECTION_SEND, Transfer::Spec::DIRECTION_RECEIVE
+      # @param ts_merge additional transfer spec to merge
       def transfer_spec_gen4(file_id, direction, ts_merge=nil)
         ak_name = nil
         ak_token = nil
@@ -288,6 +316,9 @@ module Aspera
           # TODO: token_generation_lambda = lambda{|do_refresh|oauth.token(refresh: do_refresh)}
           # get bearer token, possibly use cache
           ak_token = oauth.token
+        when :none
+          ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
+          ak_token = params[:headers]['Authorization']
         else Aspera.error_unexpected_value(auth_params[:type])
         end
         transfer_spec = {
@@ -317,13 +348,13 @@ module Aspera
           if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
             transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
           end
-          info = read('info')[:data]
+          info = read('info')
           # get the transfer user from info on access key
           transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
           # get settings from name.value array to hash key.value
           settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
           # check WSS ports
-          %w[wss_enabled wss_port].each do |i|
+          Transfer::Spec::WSS_FIELDS.each do |i|
             transfer_spec[i] = settings[i] if settings.key?(i)
           end if settings.is_a?(Hash)
         else
@@ -336,8 +367,8 @@ module Aspera
 
       private
 
+      # method called in loop for each entry for `resolve_api_fid`
       def process_api_fid(entry, path, state)
-        # this block is called recursively for each entry in folder
         # stop digging here if not in right path
         return false unless entry['name'].eql?(state[:path].first)
         # ok it matches, so we remove the match, and continue digging
@@ -380,11 +411,18 @@ module Aspera
         return true
       end
 
-      def process_find_files(entry, _path, state)
-        state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
+      # method called in loop for each entry for `find_files`
+      def process_find_files(entry, path, state)
+        state[:found].push(entry.merge({'path' => path})) if state[:test_lambda].call(entry)
         # test all files deeply
         return true
       end
+
+      # method called in loop for each entry for `list_files`
+      def process_list_files(entry, path, state)
+        state[:found].push(entry.merge({'path' => path}))
+        return false
+      end
     end
   end
 end

data/lib/aspera/ascmd.rb

@@ -22,7 +22,54 @@ module Aspera
       mv:     2,
       rm:     1
     }.freeze
-    private_constant :OPS_ARGS
+
+    # protocol is based on Type-Length-Value
+    # type start at one, but array index start at zero
+    ENUM_START = 1
+
+    # description of result structures (see ascmdtypes.h).
+    # Base types are big endian
+    # key = name of type
+    # index in array `fields` is the type (minus ENUM_START)
+    # decoding always start at `result`
+    # some fields have special handling indicated by `special`
+    # field_list, list_tlv_list, list_tlv_restart are composed with a list of TLV
+    TYPES_DESCR = {
+      result: {decode: :field_list,
+               fields: [{name: :file, is_a: :stat}, {name: :dir, is_a: :stat, special: :list_tlv_list}, {name: :size, is_a: :size}, {name: :error, is_a: :error},
+                        {name: :info, is_a: :info}, {name: :success, is_a: nil, special: :return_true}, {name: :exit, is_a: nil},
+                        {name: :df, is_a: :mnt, special: :list_tlv_restart}, {name: :md5sum, is_a: :md5sum}]},
+      stat:   {decode: :field_list,
+               fields: [{name: :name, is_a: :zstr}, {name: :size, is_a: :int64}, {name: :mode, is_a: :int32, check: nil}, {name: :zmode, is_a: :zstr},
+                        {name: :uid, is_a: :int32, check: nil}, {name: :zuid, is_a: :zstr}, {name: :gid, is_a: :int32, check: nil}, {name: :zgid, is_a: :zstr},
+                        {name: :ctime, is_a: :epoch}, {name: :zctime, is_a: :zstr}, {name: :mtime, is_a: :epoch}, {name: :zmtime, is_a: :zstr},
+                        {name: :atime, is_a: :epoch}, {name: :zatime, is_a: :zstr}, {name: :symlink, is_a: :zstr}, {name: :errno, is_a: :int32},
+                        {name: :errstr, is_a: :zstr}]},
+      info:   {decode: :field_list,
+               fields: [{name: :platform, is_a: :zstr}, {name: :version, is_a: :zstr}, {name: :lang, is_a: :zstr}, {name: :territory, is_a: :zstr},
+                        {name: :codeset, is_a: :zstr}, {name: :lc_ctype, is_a: :zstr}, {name: :lc_numeric, is_a: :zstr}, {name: :lc_time, is_a: :zstr},
+                        {name: :lc_all, is_a: :zstr}, {name: :dev, is_a: :zstr, special: :list_multiple}, {name: :browse_caps, is_a: :zstr},
+                        {name: :protocol, is_a: :zstr}]},
+      size:   {decode: :field_list,
+               fields: [{name: :size, is_a: :int64}, {name: :fcount, is_a: :int32}, {name: :dcount, is_a: :int32}, {name: :failed_fcount, is_a: :int32},
+                        {name: :failed_dcount, is_a: :int32}]},
+      error:  {decode: :field_list,
+               fields: [{name: :errno, is_a: :int32}, {name: :errstr, is_a: :zstr}]},
+      mnt:    {decode: :field_list,
+               fields: [{name: :fs, is_a: :zstr}, {name: :dir, is_a: :zstr}, {name: :is_a, is_a: :zstr}, {name: :total, is_a: :int64},
+                        {name: :used, is_a: :int64}, {name: :free, is_a: :int64}, {name: :fcount, is_a: :int64}, {name: :errno, is_a: :int32},
+                        {name: :errstr, is_a: :zstr}]},
+      md5sum: {decode: :field_list, fields: [{name: :md5sum, is_a: :zstr}]},
+      int8:   {decode: :base, unpack: 'C', size: 1},
+      int32:  {decode: :base, unpack: 'L>', size: 4},
+      int64:  {decode: :base, unpack: 'Q>', size: 8},
+      epoch:  {decode: :base, unpack: 'Q>', size: 8},
+      zstr:   {decode: :base, unpack: 'Z*'},
+      blist:  {decode: :buffer_list}
+    }.freeze
+
+    private_constant :TYPES_DESCR, :ENUM_START, :OPS_ARGS
+
     # list of supported actions
     OPERATIONS = OPS_ARGS.keys.freeze
 
@@ -55,6 +102,7 @@ module Aspera
       arg_batches.each do |args|
         command = [main_command]
         # enclose arguments in double quotes, protect backslash and double quotes
+        # ascmd uses space as token separator, and optional quotes ('") or \ to escape
         args.each do |v|
           command.push(%Q{"#{v.gsub(/["\\]/){|s|"\\#{s}"}}"})
         end
@@ -106,47 +154,6 @@ module Aspera
       def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments&.join(',')}"; end
     end
 
-    # description of result structures (see ascmdtypes.h). Base types are big endian
-    # key = name of type
-    TYPES_DESCR = {
-      result: {decode: :field_list,
-               fields: [{name: :file, is_a: :stat}, {name: :dir, is_a: :stat, special: :sub_struct}, {name: :size, is_a: :size}, {name: :error, is_a: :error},
-                        {name: :info, is_a: :info}, {name: :success, is_a: nil, special: :return_true}, {name: :exit, is_a: nil},
-                        {name: :df, is_a: :mnt, special: :restart_on_first}, {name: :md5sum, is_a: :md5sum}]},
-      stat:   {decode: :field_list,
-               fields: [{name: :name, is_a: :zstr}, {name: :size, is_a: :int64}, {name: :mode, is_a: :int32, check: nil}, {name: :zmode, is_a: :zstr},
-                        {name: :uid, is_a: :int32, check: nil}, {name: :zuid, is_a: :zstr}, {name: :gid, is_a: :int32, check: nil}, {name: :zgid, is_a: :zstr},
-                        {name: :ctime, is_a: :epoch}, {name: :zctime, is_a: :zstr}, {name: :mtime, is_a: :epoch}, {name: :zmtime, is_a: :zstr},
-                        {name: :atime, is_a: :epoch}, {name: :zatime, is_a: :zstr}, {name: :symlink, is_a: :zstr}, {name: :errno, is_a: :int32},
-                        {name: :errstr, is_a: :zstr}]},
-      info:   {decode: :field_list,
-               fields: [{name: :platform, is_a: :zstr}, {name: :version, is_a: :zstr}, {name: :lang, is_a: :zstr}, {name: :territory, is_a: :zstr},
-                        {name: :codeset, is_a: :zstr}, {name: :lc_ctype, is_a: :zstr}, {name: :lc_numeric, is_a: :zstr}, {name: :lc_time, is_a: :zstr},
-                        {name: :lc_all, is_a: :zstr}, {name: :dev, is_a: :zstr, special: :multiple}, {name: :browse_caps, is_a: :zstr},
-                        {name: :protocol, is_a: :zstr}]},
-      size:   {decode: :field_list,
-               fields: [{name: :size, is_a: :int64}, {name: :fcount, is_a: :int32}, {name: :dcount, is_a: :int32}, {name: :failed_fcount, is_a: :int32},
-                        {name: :failed_dcount, is_a: :int32}]},
-      error:  {decode: :field_list,
-               fields: [{name: :errno, is_a: :int32}, {name: :errstr, is_a: :zstr}]},
-      mnt:    {decode: :field_list,
-               fields: [{name: :fs, is_a: :zstr}, {name: :dir, is_a: :zstr}, {name: :is_a, is_a: :zstr}, {name: :total, is_a: :int64},
-                        {name: :used, is_a: :int64}, {name: :free, is_a: :int64}, {name: :fcount, is_a: :int64}, {name: :errno, is_a: :int32},
-                        {name: :errstr, is_a: :zstr}]},
-      md5sum: {decode: :field_list, fields: [{name: :md5sum, is_a: :zstr}]},
-      int8:   {decode: :base, unpack: 'C', size: 1},
-      int32:  {decode: :base, unpack: 'L>', size: 4},
-      int64:  {decode: :base, unpack: 'Q>', size: 8},
-      epoch:  {decode: :base, unpack: 'Q>', size: 8},
-      zstr:   {decode: :base, unpack: 'Z*'},
-      blist:  {decode: :buffer_list}
-    }.freeze
-
-    # protocol enum start at one, but array index start at zero
-    ENUM_START = 1
-
-    private_constant :TYPES_DESCR, :ENUM_START
-
     class << self
       # get description of structure's field, @param struct_name, @param typed_buffer provides field name
       def field_description(struct_name, typed_buffer)
@@ -175,6 +182,7 @@ module Aspera
           Log.log.trace1{"#{'   .' * indent_level}-> base:#{byte_array} -> #{result}"}
           result = Time.at(result) if type_name.eql?(:epoch)
         when :buffer_list
+          # return a list of type_buffer
           result = []
           until buffer.empty?
             btype = parse(buffer, :int8, indent_level)
@@ -193,16 +201,16 @@ module Aspera
             field_info = field_description(type_name, typed_buffer)
             Log.log.trace1{"#{'   .' * indent_level}+ field(special=#{field_info[:special]})=#{field_info[:name]}".green}
             case field_info[:special]
-            when nil
+            when nil # normal case
               result[field_info[:name]] = parse(typed_buffer[:buffer], field_info[:is_a], indent_level)
-            when :return_true
+            when :return_true # nothing to parse, just return true
               result[field_info[:name]] = true
-            when :sub_struct
-              result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{|r|parse(r[:buffer], field_info[:is_a], indent_level)}
-            when :multiple
+            when :list_multiple # field appears multiple times, and is an array of values (base type)
               result[field_info[:name]] ||= []
               result[field_info[:name]].push(parse(typed_buffer[:buffer], field_info[:is_a], indent_level))
-            when :restart_on_first
+            when :list_tlv_list # field is an array of values in a list of buffers
+              result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{|r|parse(r[:buffer], field_info[:is_a], indent_level)}
+            when :list_tlv_restart # field is an array of values, but a new value is started on index 1
               fl = result[field_info[:name]] = []
               parse(typed_buffer[:buffer], :blist, indent_level).map do |tb|
                 fl.push({}) if tb[:btype].eql?(ENUM_START)