aspera-cli 4.18.0 → 4.19.0

data/lib/aspera/api/httpgw.rb

@@ -45,7 +45,7 @@ module Aspera
         else
           @shared_info[:count][:sent_general] += 1
         end
-        Log.log.debug do
+        Log.log.trace1 do
           log_data = payload.dup
           log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
           "#{LOG_WS_SEND}json: #{msg_type}: #{JSON.generate(log_data)}"
@@ -55,7 +55,7 @@ module Aspera
 
       # send data on http gw web socket
       def ws_send(ws_type:, data:)
-        Log.log.debug{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
+        Log.log.trace1{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
         @shared_info[:count][:sent_general] += 1 if ws_type.eql?(:binary)
         frame_generator = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: ws_type, version: @ws_handshake.version)
         @ws_io.write(frame_generator.to_s)
@@ -68,13 +68,13 @@ module Aspera
                 (((@shared_info[:count][:sent_general] - @shared_info[:count][:received_general]) > 1) ||
                   ((@shared_info[:count][:received_v2_delimiter] - @shared_info[:count][:sent_v2_delimiter]) > 1))
               if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
-                Log.log.debug{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
+                Log.log.trace1{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
               end
             end
           end
         end
         raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
-        Log.log.debug{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
+        Log.log.trace2{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
       end
 
       # message processing for read thread
@@ -114,12 +114,12 @@ module Aspera
             # ready byte by byte until frame is ready
             # blocking read
             byte = @ws_io.read(1)
-            Log.log.trace1{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
+            Log.log.trace2{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
             frame_parser << byte
             frame_ok = frame_parser.next
             next if frame_ok.nil?
             process_received_message(frame_ok.data.to_s)
-            Log.log.debug{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
+            Log.log.trace2{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
           rescue => e
             Log.log.debug{"#{LOG_WS_RECV}Exception: #{e}"}
             @shared_info[:mutex].synchronize do
@@ -139,34 +139,12 @@ module Aspera
         # identify this session uniquely
         session_id = SecureRandom.uuid
         @notify_cb&.call(session_id: nil, type: :pre_start, info: 'starting')
-        # total size of all files
-        total_bytes_to_transfer = 0
-        # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
-        files_to_read = []
-        # get source root or nil
-        source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
-        # source root is ignored by GW, used only here
-        transfer_spec.delete('source_root')
-        # compute total size of files to upload (for progress)
-        # modify transfer spec to be suitable for GW
-        transfer_spec['paths'].each do |item|
-          # save actual file location to be able read contents later
-          file_to_add = Transfer::FauxFile.open(item['source'])
-          if file_to_add
-            item['source'] = file_to_add.path
-            item['file_size'] = file_to_add.size
-          else
-            file_to_add = item['source']
-            # add source root if needed
-            file_to_add = File.join(source_root, file_to_add) unless source_root.nil?
-            # GW expects a simple file name in 'source' but if user wants to change the name, we take it
-            item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
-            item['file_size'] = File.size(file_to_add)
-          end
-          # save so that we can actually read the file later
-          files_to_read.push(file_to_add)
-          total_bytes_to_transfer += item['file_size']
-        end
+        # process files to send, modify `paths` in transfer_spec
+        files_to_send = process_upload_list(transfer_spec)
+        # total size of all files is last element
+        total_bytes_to_transfer = files_to_send.pop
+        Log.log.trace1{Log.dump(:modified_tspec, transfer_spec)}
+        Log.log.trace1{Log.dump(:files_to_send, files_to_send)}
         # TODO: check that this is available in endpoints: @api_info['endpoints']
         upload_url = File.join(@gw_root_url, @upload_version, 'upload')
         @notify_cb&.call(session_id: nil, type: :pre_start, info: 'connecting wss')
@@ -180,11 +158,6 @@ module Aspera
         @ws_handshake << @ws_io.readuntil("\r\n\r\n")
         Aspera.assert(@ws_handshake.finished?){'Error in websocket handshake'}
         Log.log.debug{"#{LOG_WS_SEND}handshake success"}
-        # start read thread after handshake
-        @ws_read_thread = Thread.new {process_read_thread}
-        @notify_cb&.call(session_id: session_id, type: :session_start)
-        @notify_cb&.call(session_id: session_id, type: :session_size, info: total_bytes_to_transfer)
-        sleep(1)
         # data shared between main thread and read thread
         @shared_info = {
           read_exception: nil, # error message if any in callback
@@ -197,34 +170,34 @@ module Aspera
           mutex:          Mutex.new,
           cond_var:       ConditionVariable.new
         }
+        # start read thread after handshake
+        @ws_read_thread = Thread.new {process_read_thread}
+        @notify_cb&.call(session_id: session_id, type: :session_start)
+        @notify_cb&.call(session_id: session_id, type: :session_size, info: total_bytes_to_transfer)
+        sleep(1)
         # notify progress bar
         @notify_cb&.call(type: :session_size, session_id: session_id, info: total_bytes_to_transfer)
         # first step send transfer spec
-        Log.log.debug{Log.dump(:ws_spec, transfer_spec)}
         ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
         # current file index
         file_index = 0
         # aggregate size sent
         session_sent_bytes = 0
         # process each file
-        transfer_spec['paths'].each do |item|
+        files_to_send.each do |file_to_send|
+          last_slice = (file_to_send[:size] - 1) / @upload_chunk_size
           slice_info = {
-            name:       nil,
+            name:         file_to_send[:name],
             # TODO: get mime type?
-            type:       'application/octet-stream',
-            size:       item['file_size'],
-            slice:      0, # current slice index
+            type:         'application/octet-stream',
+            size:         file_to_send[:size],
+            slice:        0, # current slice index
             # index of last slice (i.e number of slices - 1)
-            last_slice: (item['file_size'] - 1) / @upload_chunk_size,
-            fileIndex:  file_index
+            total_slices: last_slice + 1,
+            fileIndex:    file_index
           }
-          file = files_to_read[file_index]
-          if file.is_a?(Transfer::FauxFile)
-            slice_info[:name] = file.path
-          else
-            file = File.open(file)
-            slice_info[:name] = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
-          end
+          file = file_to_send[:file]
+          file = File.open(file) unless file.is_a?(Transfer::FauxFile)
           begin
             until file.eof?
               slice_bin_data = file.read(@upload_chunk_size)
@@ -238,9 +211,9 @@ module Aspera
                   # send once, before data, at beginning
                   ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(0)
                   ws_send(ws_type: :binary, data: slice_bin_data)
-                  Log.log.debug{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{slice_info[:last_slice]}"}
+                  Log.log.trace1{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{last_slice}"}
                   # send once, after data, at end
-                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(slice_info[:last_slice])
+                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(last_slice)
                 end
               rescue Errno::EPIPE => e
                 raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
@@ -310,11 +283,14 @@ module Aspera
         notify_cb:         nil,
         **opts
       )
+        Log.log.debug{Log.dump(:gw_url, url)}
         # add scheme if missing
         url = "https://#{url}" unless url.match?(%r{^[a-z]{1,6}://})
         raise 'GW URL shall be with scheme https' unless url.start_with?('https://')
-        # remove trailing slash and version if any
+        # remove trailing slash and version (o=only once) if present
+        # TODO: issue warning ?
         url = url.gsub(%r{/+$}, '').gsub(%r{/#{API_V1}$}o, '')
+        # assume GW is always under specific path (TODO: remove this ?)
         url = File.join(url, DEFAULT_BASE_PATH) unless url.end_with?(DEFAULT_BASE_PATH)
         @gw_root_url = url
         super(base_url: "#{@gw_root_url}/#{API_V1}", **opts)
@@ -334,6 +310,68 @@ module Aspera
           end
         end
       end
+
+      private
+
+      # compute total size of files to upload (for progress)
+      # modify transfer spec to be suitable for HTTPGW
+      # @param transfer_spec [Hash] transfer specification
+      # @return [Array] info on files to send
+      def process_upload_list(transfer_spec)
+        total_bytes_to_transfer = 0
+        source_prefix = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] + '/' : ''
+        files_to_send = []
+        transfer_spec['paths'].each do |one_path|
+          source_path = source_prefix + one_path['source']
+          faux_file = Transfer::FauxFile.create(source_path)
+          if faux_file
+            total_bytes_to_transfer += faux_file.size
+            files_to_send.push({
+              file: faux_file,
+              name: faux_file.path,
+              size: faux_file.size
+            })
+          elsif File.file?(source_path)
+            # regular file
+            file_size = File.size(source_path)
+            total_bytes_to_transfer += file_size
+            files_to_send.push({
+              file: source_path,
+              # GW expects a simple file name in 'source' but if user wants to change the name, we take it
+              name: File.basename(one_path['destination'].nil? ? source_path : one_path['destination']),
+              size: file_size
+            })
+          elsif File.directory?(source_path)
+            folders_to_process = [source_path]
+            until folders_to_process.empty?
+              folder = folders_to_process.shift
+              # read all entries
+              Dir.entries(folder).each do |entry|
+                next if entry.eql?('.') || entry.eql?('..')
+                entry_path = File.join(folder, entry)
+                if File.directory?(entry_path)
+                  folders_to_process.push(entry_path)
+                elsif File.file?(entry_path)
+                  file_size = File.size(entry_path)
+                  total_bytes_to_transfer += file_size
+                  files_to_send.push({
+                    file: entry_path,
+                    name: entry_path,
+                    size: file_size
+                  })
+                else
+                  Log.log.warn{"Ignoring non file/directory: #{entry_path}"}
+                end
+              end
+            end
+          else
+            raise "File not found: #{source_path}"
+          end
+        end
+        transfer_spec['paths'] = files_to_send.map{|i|{'source' => i[:name]}}
+        files_to_send.push(total_bytes_to_transfer)
+        return files_to_send
+      end
     end
   end
 end

data/lib/aspera/api/node.rb

@@ -14,23 +14,28 @@ module Aspera
   module Api
     # Provides additional functions using node API with gen4 extensions (access keys)
     class Node < Aspera::Rest
-      # node api permissions
-      ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
-      HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
       SCOPE_SEPARATOR = ':'
-      SCOPE_USER = 'user:all'
-      SCOPE_ADMIN = 'admin:all'
       SCOPE_NODE_PREFIX = 'node.'
       # prefix for ruby code for filter (deprecated)
       MATCH_EXEC_PREFIX = 'exec:'
       MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
-      PATH_SEPARATOR = '/'
       SIGNATURE_DELIMITER = '==SIGNATURE=='
       BEARER_TOKEN_VALIDITY_DEFAULT = 86400
-      BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
-      private_constant :MATCH_EXEC_PREFIX, :MATCH_TYPES,
-        :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT, :BEARER_TOKEN_SCOPE_DEFAULT,
-        :SCOPE_SEPARATOR, :SCOPE_NODE_PREFIX
+      # fields in @app_info
+      REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
+      # methods of @app_info[:api]
+      REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
+      private_constant :SCOPE_SEPARATOR, :SCOPE_NODE_PREFIX, :MATCH_EXEC_PREFIX, :MATCH_TYPES,
+        :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT,
+        :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
+
+      # node api permissions
+      ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
+      HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
+      HEADER_X_TOTAL_COUNT = 'X-Total-Count'
+      SCOPE_USER = 'user:all'
+      SCOPE_ADMIN = 'admin:all'
+      PATH_SEPARATOR = '/'
 
       # register node special token decoder
       OAuth::Factory.instance.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
@@ -59,7 +64,7 @@ module Aspera
         end
 
         def file_matcher_from_argument(options)
-          return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
+          return file_matcher(options.get_next_argument('filter', validation: MATCH_TYPES, mandatory: false))
         end
 
         # node API scopes
@@ -86,7 +91,7 @@ module Aspera
           # manage convenience parameters
           expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
           payload.delete('_validity')
-          scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
+          scope = payload['_scope'] || SCOPE_USER
           payload.delete('_scope')
           payload['scope'] ||= token_scope(access_key, scope)
           payload['auth_type'] ||= 'access_key'
@@ -117,19 +122,13 @@ module Aspera
         end
       end
 
-      # fields in @app_info
-      REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
-      # methods of @app_info[:api]
-      REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
-      private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
-
       attr_reader :app_info
 
+      # @param app_info  [Hash,NilClass]   Special processing for AoC
+      # @param add_tspec [Hash,NilClass]   Additional transfer spec
       # @param base_url  [String]          Rest parameters
       # @param auth      [String,NilClass] Rest parameters
       # @param headers   [String,NilClass] Rest parameters
-      # @param app_info  [Hash,NilClass]   Special processing for AoC
-      # @param add_tspec [Hash,NilClass]   Additional transfer spec
       def initialize(app_info: nil, add_tspec: nil, **rest_args)
         # init Rest
         super(**rest_args)
@@ -162,25 +161,42 @@ module Aspera
             workspace_id: @app_info[:workspace_id],
             workspace_name: @app_info[:workspace_name])
         end
-        Log.log.warn{"cannot resolve link with node id #{node_id}"}
+        Log.log.warn{"Cannot resolve link with node id #{node_id}, no resolver"}
         return nil
       end
 
+      # Check if a link entry in folder has target information
+      # @param entry [Hash] entry in folder
+      # @return [Boolean] true if target information is available
+      def entry_has_link_information(entry)
+        # if target information is missing in folder, try to get it on entry
+        if entry['target_node_id'].nil? || entry['target_id'].nil?
+          link_entry = read("files/#{entry['id']}")[:data]
+          entry['target_node_id'] = link_entry['target_node_id']
+          entry['target_id'] = link_entry['target_id']
+        end
+        return true unless entry['target_node_id'].nil? || entry['target_id'].nil?
+        Log.log.warn{"Missing target information for link: #{entry['name']}"}
+        return false
+      end
+
       # Recursively browse in a folder (with non-recursive method)
       # sub folders are processed if the processing method returns true
+      # links are processed on the respective node
       # @param state [Object] state object sent to processing method
       # @param top_file_id [String] file id to start at (default = access key root file id)
       # @param top_file_path [String] path of top folder (default = /)
       # @param block [Proc] processing method, arguments: entry, path, state
-      def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
+      def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/')
         Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
-        Aspera.assert(block){'Missing block'}
+        Log.log.debug{"process_folder_tree: node=#{@app_info ? @app_info[:node_info]['id'] : 'nil'}, file id=#{top_file_id},  path=#{top_file_path}"}
         # start at top folder
         folders_to_explore = [{id: top_file_id, path: top_file_path}]
         Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
         until folders_to_explore.empty?
+          # consume first in job list
           current_item = folders_to_explore.shift
-          Log.log.debug{"searching #{current_item[:path]}".bg_green}
+          Log.log.debug{"Exploring #{current_item[:path]}".bg_green}
           # get folder content
           folder_contents =
             begin
@@ -192,19 +208,21 @@ module Aspera
           Log.log.debug{Log.dump(:folder_contents, folder_contents)}
           folder_contents.each do |entry|
             relative_path = File.join(current_item[:path], entry['name'])
-            Log.log.debug{"process_folder_tree checking #{relative_path}"}
-            # continue only if method returns true
-            next unless yield(entry, relative_path, state)
+            Log.log.debug{"process_folder_tree: checking #{relative_path}"}
+            # call block, continue only if method returns true
+            next unless send(method_sym, entry, relative_path, state)
             # entry type is file, folder or link
             case entry['type']
             when 'folder'
               folders_to_explore.push({id: entry['id'], path: relative_path})
             when 'link'
-              node_id_to_node(entry['target_node_id'])&.process_folder_tree(
-                state:         state,
-                top_file_id:   entry['target_id'],
-                top_file_path: relative_path,
-                &block)
+              if entry_has_link_information(entry)
+                node_id_to_node(entry['target_node_id'])&.process_folder_tree(
+                  method_sym:    method_sym,
+                  state:         state,
+                  top_file_id:   entry['target_id'],
+                  top_file_path: relative_path)
+              end
             end
           end
         end
@@ -216,60 +234,23 @@ module Aspera
       # @return [Hash] {.api,.file_id}
       def resolve_api_fid(top_file_id, path)
         Aspera.assert_type(top_file_id, String)
+        Aspera.assert_type(path, String)
+        # if last element is a link and followed by "/", we list the content of that folder, else we return the link
         process_last_link = path.end_with?(PATH_SEPARATOR)
+        # keep only non-empty elements
         path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
         return {api: self, file_id: top_file_id} if path_elements.empty?
-        resolve_state = {path: path_elements, result: nil}
-        process_folder_tree(state: resolve_state, top_file_id: top_file_id) do |entry, _path, state|
-          # this block is called recursively for each entry in folder
-          # stop digging here if not in right path
-          next false unless entry['name'].eql?(state[:path].first)
-          # ok it matches, so we remove the match
-          state[:path].shift
-          case entry['type']
-          when 'file'
-            # file must be terminal
-            raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
-            # it's terminal, we found it
-            state[:result] = {api: self, file_id: entry['id']}
-            next false
-          when 'folder'
-            if state[:path].empty?
-              # we found it
-              state[:result] = {api: self, file_id: entry['id']}
-              next false
-            end
-          when 'link'
-            if state[:path].empty?
-              if process_last_link
-                # we found it
-                other_node = node_id_to_node(entry['target_node_id'])
-                raise 'cannot resolve link' if other_node.nil?
-                state[:result] = {api: other_node, file_id: entry['target_id']}
-              else
-                # we found it but we do not process the link
-                state[:result] = {api: self, file_id: entry['id']}
-              end
-              next false
-            end
-          else
-            Log.log.warn{"Unknown element type: #{entry['type']}"}
-          end
-          # continue to dig folder
-          next true
-        end
+        resolve_state = {path: path_elements, result: nil, process_last_link: process_last_link}
+        process_folder_tree(method_sym: :process_api_fid, state: resolve_state, top_file_id: top_file_id)
         raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
+        Log.log.debug{"resolve_api_fid: #{path} -> #{resolve_state[:result][:api].base_url} #{resolve_state[:result][:file_id]}"}
         return resolve_state[:result]
       end
 
       def find_files(top_file_id, test_block)
         Log.log.debug{"find_files: file id=#{top_file_id}"}
         find_state = {found: [], test_block: test_block}
-        process_folder_tree(state: find_state, top_file_id: top_file_id) do |entry, path, state|
-          state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
-          # test all files deeply
-          true
-        end
+        process_folder_tree(method_sym: :process_find_files, state: find_state, top_file_id: top_file_id)
         return find_state[:found]
       end
 
@@ -306,7 +287,7 @@ module Aspera
           ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
           # TODO: token_generation_lambda = lambda{|do_refresh|oauth.token(refresh: do_refresh)}
           # get bearer token, possibly use cache
-          ak_token = oauth.token(refresh: false)
+          ak_token = oauth.token
         else Aspera.error_unexpected_value(auth_params[:type])
         end
         transfer_spec = {
@@ -352,6 +333,58 @@ module Aspera
           unless transfer_spec['remote_user'].eql?(Transfer::Spec::ACCESS_KEY_TRANSFER_USER)
         return transfer_spec
       end
+
+      private
+
+      def process_api_fid(entry, path, state)
+        # this block is called recursively for each entry in folder
+        # stop digging here if not in right path
+        return false unless entry['name'].eql?(state[:path].first)
+        # ok it matches, so we remove the match, and continue digging
+        state[:path].shift
+        path_fully_consumed = state[:path].empty?
+        case entry['type']
+        when 'file'
+          # file must be terminal
+          raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless path_fully_consumed
+          # it's terminal, we found it
+          Log.log.debug{"resolve_api_fid: found #{path} -> #{entry['id']}"}
+          state[:result] = {api: self, file_id: entry['id']}
+          return false
+        when 'folder'
+          if path_fully_consumed
+            # we found it
+            state[:result] = {api: self, file_id: entry['id']}
+            return false
+          end
+        when 'link'
+          if path_fully_consumed
+            if state[:process_last_link]
+              # we found it
+              other_node = nil
+              if entry_has_link_information(entry)
+                other_node = node_id_to_node(entry['target_node_id'])
+              end
+              raise 'Cannot resolve link' if other_node.nil?
+              state[:result] = {api: other_node, file_id: entry['target_id']}
+            else
+              # we found it but we do not process the link
+              state[:result] = {api: self, file_id: entry['id']}
+            end
+            return false
+          end
+        else
+          Log.log.warn{"Unknown element type: #{entry['type']}"}
+        end
+        # continue to dig folder
+        return true
+      end
+
+      def process_find_files(entry, _path, state)
+        state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
+        # test all files deeply
+        return true
+      end
     end
   end
 end