aspera-cli 4.2.2 → 4.3.0

data/lib/aspera/aoc.rb

@@ -273,7 +273,7 @@ module Aspera
         transfer_spec['remote_host']=node_file[:node_info]['host']
       else
         # retrieve values from API
-        std_t_spec=get_node_api(node_file[:node_info],SCOPE_NODE_USER).create('files/download_setup',{:transfer_requests => [ { :transfer_request => {:paths => [ {"source"=>'/'} ] } } ] } )[:data]['transfer_specs'].first['transfer_spec']
+        std_t_spec=get_node_api(node_file[:node_info],scope: SCOPE_NODE_USER).create('files/download_setup',{:transfer_requests => [ { :transfer_request => {:paths => [ {"source"=>'/'} ] } } ] } )[:data]['transfer_specs'].first['transfer_spec']
         ['remote_host','remote_user','ssh_port','fasp_port'].each {|i| transfer_spec[i]=std_t_spec[i]}
       end
       # add caller provided transfer spec
@@ -290,26 +290,27 @@ module Aspera
     # @param scope e.g. SCOPE_NODE_USER
     # no scope: requires secret
     # if secret provided beforehand: use it
-    def get_node_api(node_info,node_scope=nil)
-      # X-Aspera-AccessKey required for bearer token only
-      node_rest_params={
-        :base_url => node_info['url'],
-        :headers  => {'X-Aspera-AccessKey'=>node_info['access_key']},
-      }
-      ak_secret=@key_chain.get_secret(node_info['access_key'],false)
-      if ak_secret.nil? and node_scope.nil?
+    def get_node_api(node_info,options={})
+      raise "INTERNAL ERROR: method parameters: options must ne hash" unless options.is_a?(Hash)
+      options.keys.each {|k| raise "INTERNAL ERROR: not valid option: #{k}" unless [:scope,:use_secret].include?(k)}
+      # get optional secret unless :use_secret is false (default is true)
+      ak_secret=@key_chain.get_secret(node_info['access_key'],false) if !options.has_key?(:use_secret) or options[:use_secret]
+      if ak_secret.nil? and !options.has_key?(:scope)
         raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}"
       end
-      # if secret provided on command line or if there is no scope
-      if !ak_secret.nil? or node_scope.nil?
+      node_rest_params={base_url: node_info['url']}
+      # if secret is available
+      if !ak_secret.nil?
         node_rest_params[:auth]={
-          :type     => :basic,
-          :username => node_info['access_key'],
-          :password => ak_secret
+          type:     :basic,
+          username: node_info['access_key'],
+          password: ak_secret
         }
       else
+        # X-Aspera-AccessKey required for bearer token only
+        node_rest_params[:headers]= {'X-Aspera-AccessKey'=>node_info['access_key']}
         node_rest_params[:auth]=self.params[:auth].clone
-        node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],node_scope)
+        node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],options[:scope])
       end
       return Node.new(node_rest_params)
     end
@@ -336,7 +337,7 @@ module Aspera
         if entry[:type].eql?('link')
           sub_node_info=self.read("nodes/#{entry['target_node_id']}")[:data]
           sub_opt={method: process_find_files, top_file_id: entry['target_id'], top_file_path: path}
-          get_node_api(sub_node_info,SCOPE_NODE_USER).crawl(self,sub_opt)
+          get_node_api(sub_node_info,scope: SCOPE_NODE_USER).crawl(self,sub_opt)
         end
       rescue => e
         Log.log.error("#{path}: #{e.message}")
@@ -349,7 +350,7 @@ module Aspera
       top_node_info,top_file_id=check_get_node_file(top_node_file)
       Log.log.debug("find_files: node_info=#{top_node_info}, fileid=#{top_file_id}")
       @find_state={found: [], test_block: test_block}
-      get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
+      get_node_api(top_node_info,scope: SCOPE_NODE_USER).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
       result=@find_state[:found]
       @find_state=nil
       return result
@@ -370,7 +371,7 @@ module Aspera
         if @resolve_state[:path].empty?
           @resolve_state[:result][:file_id]=entry['target_id']
         else
-          get_node_api(@resolve_state[:result][:node_info],SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
+          get_node_api(@resolve_state[:result][:node_info],scope: SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
         end
       when 'folder'
         if @resolve_state[:path].empty?
@@ -397,7 +398,7 @@ module Aspera
         result[:file_id]=top_file_id
       else
         @resolve_state={path: path_elements, result: result}
-        get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
+        get_node_api(top_node_info,scope: SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
         not_found=@resolve_state[:path]
         @resolve_state=nil
         raise "entry not found: #{not_found}" if result[:file_id].nil?

data/lib/aspera/cli/plugins/aoc.rb

@@ -5,6 +5,7 @@ require 'aspera/cli/transfer_agent'
 require 'aspera/aoc'
 require 'aspera/node'
 require 'aspera/persistency_action_once'
+require 'aspera/id_generator'
 require 'securerandom'
 require 'resolv'
 require 'date'
@@ -82,19 +83,29 @@ module Aspera
           return self.transfer.start(*@api_aoc.tr_spec(app,direction,node_file,ts_add))
         end
 
-        NODE4_COMMANDS=[ :browse, :find, :mkdir, :rename, :delete, :upload, :download, :transfer, :http_node_download, :v3, :file, :bearer_token_node ]
+        NODE4_COMMANDS=[ :browse, :find, :mkdir, :rename, :delete, :upload, :download, :transfer, :http_node_download, :v3, :file, :bearer_token_node, :node_info ]
 
         def execute_node_gen4_command(command_repo,top_node_file)
           case command_repo
           when :bearer_token_node
             thepath=self.options.get_next_argument('path')
             node_file = @api_aoc.resolve_node_file(top_node_file,thepath)
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER, use_secret: false)
             return Main.result_status(node_api.oauth_token)
+          when :node_info
+            thepath=self.options.get_next_argument('path')
+            node_file = @api_aoc.resolve_node_file(top_node_file,thepath)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER, use_secret: false)
+            return {:type=>:single_object,:data=>{
+              url: node_file[:node_info]['url'],
+              username: node_file[:node_info]['access_key'],
+              password: node_api.oauth_token,
+              root_id: node_file[:file_id]
+              }}
           when :browse
             thepath=self.options.get_next_argument('path')
             node_file = @api_aoc.resolve_node_file(top_node_file,thepath)
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             file_info = node_api.read("files/#{node_file[:file_id]}")[:data]
             if file_info['type'].eql?('folder')
               result=node_api.read("files/#{node_file[:file_id]}/files",self.options.get_option(:value,:optional))
@@ -114,14 +125,14 @@ module Aspera
             containing_folder_path = thepath.split(AoC::PATH_SEPARATOR)
             new_folder=containing_folder_path.pop
             node_file = @api_aoc.resolve_node_file(top_node_file,containing_folder_path.join(AoC::PATH_SEPARATOR))
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             result=node_api.create("files/#{node_file[:file_id]}/files",{:name=>new_folder,:type=>:folder})[:data]
             return Main.result_status("created: #{result['name']} (id=#{result['id']})")
           when :rename
             thepath=self.options.get_next_argument('source path')
             newname=self.options.get_next_argument('new name')
             node_file = @api_aoc.resolve_node_file(top_node_file,thepath)
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             result=node_api.update("files/#{node_file[:file_id]}",{:name=>newname})[:data]
             return Main.result_status("renamed #{thepath} to #{newname}")
           when :delete
@@ -129,7 +140,7 @@ module Aspera
             return do_bulk_operation(thepath,'deleted','path') do |l_path|
               raise "expecting String (path), got #{l_path.class.name} (#{l_path})" unless l_path.is_a?(String)
               node_file = @api_aoc.resolve_node_file(top_node_file,l_path)
-              node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+              node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
               result=node_api.delete("files/#{node_file[:file_id]}")[:data]
               {'path'=>l_path}
             end
@@ -152,7 +163,7 @@ module Aspera
             client_node_file = @api_aoc.resolve_node_file(client_home_node_file,client_folder)
             server_node_file = @api_aoc.resolve_node_file(server_home_node_file,server_folder)
             # force node as transfer agent
-            @agents[:transfer].set_agent_instance(Fasp::Node.new(@api_aoc.get_node_api(client_node_file[:node_info],AoC::SCOPE_NODE_USER)))
+            @agents[:transfer].set_agent_instance(Fasp::Node.new(@api_aoc.get_node_api(client_node_file[:node_info],scope: AoC::SCOPE_NODE_USER)))
             # additional node to node TS info
             add_ts={
               'remote_access_key'   => server_node_file[:node_info]['access_key'],
@@ -190,14 +201,14 @@ module Aspera
             raise CliBadArgument,'one file at a time only in HTTP mode' if source_paths.length > 1
             file_name = source_paths.first['source']
             node_file = @api_aoc.resolve_node_file(top_node_file,File.join(source_folder,file_name))
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             node_api.call({:operation=>'GET',:subpath=>"files/#{node_file[:file_id]}/content",:save_to_file=>File.join(self.transfer.destination_folder('receive'),file_name)})
             return Main.result_status("downloaded: #{file_name}")
           when :v3
             # Note: other "common" actions are unauthorized with user scope
             command_legacy=self.options.get_next_command(Node::SIMPLE_ACTIONS)
             # TODO: shall we support all methods here ? what if there is a link ?
-            node_api=@api_aoc.get_node_api(top_node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(top_node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: node_api)).execute_action(command_legacy)
           when :file
             file_path=self.options.get_option(:path,:optional)
@@ -206,7 +217,7 @@ module Aspera
             else
               {node_info: top_node_file[:node_info],file_id: self.options.get_option(:id,:mandatory)}
             end
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             command_node_file=self.options.get_next_command([:show,:permission,:modify])
             case command_node_file
             when :show
@@ -517,7 +528,7 @@ module Aspera
                 startdate_persistency=PersistencyActionOnce.new(
                 manager: @agents[:persistency],
                 data: saved_date,
-                ids:  ['aoc_ana_date',self.options.get_option(:url,:mandatory),@workspace_name].push(filter_resource,filter_id))
+                ids: IdGenerator.from_list(['aoc_ana_date',self.options.get_option(:url,:mandatory),@workspace_name].push(filter_resource,filter_id)))
                 start_datetime=saved_date.first
                 stop_datetime=Time.now.utc.strftime('%FT%T.%LZ')
                 #Log.log().error("start: #{start_datetime}")
@@ -653,7 +664,7 @@ module Aspera
               res_data=@api_aoc.read(resource_instance_path)[:data]
               node_file = @api_aoc.resolve_node_file({node_info: res_data, file_id: ak_data['root_file_id']},folder_path)
 
-              #node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+              #node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
               #file_info = node_api.read("files/#{node_file[:file_id]}")[:data]
 
               access_id="ASPERA_ACCESS_KEY_ADMIN_WS_#{@workspace_id}"
@@ -781,7 +792,7 @@ module Aspera
                 skip_ids_persistency=PersistencyActionOnce.new(
                 manager: @agents[:persistency],
                 data: skip_ids_data,
-                ids:  ['aoc_recv',self.options.get_option(:url,:mandatory),@workspace_id].push(*@persist_ids))
+                id: IdGenerator.from_list(['aoc_recv',self.options.get_option(:url,:mandatory),@workspace_id].push(*@persist_ids)))
               end
               if ids_to_download.eql?(VAL_ALL)
                 # get list of packages in inbox
@@ -875,7 +886,7 @@ module Aspera
               end
               result=self.entity_action(@api_aoc,'short_links',nil,:id,'self')
               if result[:data].is_a?(Hash) and result[:data].has_key?('created_at') and result[:data]['resource_type'].eql?('UrlToken')
-                node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+                node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
                 # TODO: access level as arg
                 access_levels=Aspera::Node::ACCESS_LEVELS #['delete','list','mkdir','preview','read','rename','write']
                 perm_data={

data/lib/aspera/cli/plugins/config.rb

@@ -9,6 +9,7 @@ require 'aspera/proxy_auto_config'
 require 'aspera/uri_reader'
 require 'aspera/rest'
 require 'aspera/persistency_action_once'
+require 'aspera/id_generator'
 require 'xmlsimple'
 require 'base64'
 require 'net/smtp'
@@ -148,7 +149,7 @@ END_OF_TEMPLATE
             check_date_persist=PersistencyActionOnce.new(
             manager: persistency,
             data:    last_check_array,
-            ids:     ['version_last_check'])
+            id:      'version_last_check')
             # get persisted date or nil
             last_check_date = begin
               Date.strptime(last_check_array.first, '%Y/%m/%d')

data/lib/aspera/cli/plugins/faspex.rb

@@ -7,6 +7,7 @@ require 'aspera/persistency_action_once'
 require 'aspera/open_application'
 require 'aspera/fasp/uri'
 require 'aspera/nagios'
+require 'aspera/id_generator'
 require 'xmlsimple'
 require 'json'
 require 'cgi'
@@ -260,8 +261,8 @@ module Aspera
                 if self.options.get_option(:once_only,:mandatory)
                   skip_ids_persistency=PersistencyActionOnce.new(
                   manager: @agents[:persistency],
-                  data: skip_ids_data,
-                  ids:  ['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s])
+                  data:    skip_ids_data,
+                  id:      IdGenerator.from_list(['faspex_recv',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),self.options.get_option(:box,:mandatory).to_s]))
                 end
                 # get command line parameters
                 delivid=self.options.get_option(:id,:mandatory)

data/lib/aspera/cli/plugins/faspex5.rb

@@ -1,5 +1,6 @@
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/persistency_action_once'
+require 'aspera/id_generator'
 require 'securerandom'
 
 module Aspera
@@ -103,8 +104,8 @@ module Aspera
                 # read ids from persistency
                 skip_ids_persistency=PersistencyActionOnce.new(
                 manager: @agents[:persistency],
-                data: skip_ids_data,
-                ids:  ['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type])
+                data:    skip_ids_data,
+                id:      IdGenerator.from_list(['faspex_recv',options.get_option(:url,:mandatory),options.get_option(:username,:mandatory),pkg_type]))
               end
               if pack_id.eql?(VAL_ALL)
                 # TODO: if packages have same name, they will overwrite

data/lib/aspera/cli/plugins/node.rb

@@ -1,6 +1,7 @@
 require 'aspera/cli/basic_auth_plugin'
 require 'aspera/nagios'
 require 'aspera/hash_ext'
+require 'aspera/id_generator'
 require 'base64'
 require 'zlib'
 
@@ -271,8 +272,8 @@ module Aspera
             if self.options.get_option(:once_only,:mandatory)
               skip_ids_persistency=PersistencyActionOnce.new(
               manager: @agents[:persistency],
-              data: iteration_data,
-              ids:  ['sync_files',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),asyncid])
+              data:    iteration_data,
+              id:      IdGenerator.from_list(['sync_files',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory),asyncid]))
               unless iteration_data.first.nil?
                 data.select!{|l| l['fnid'].to_i>iteration_data.first}
               end

data/lib/aspera/cli/plugins/preview.rb

@@ -6,6 +6,8 @@ require 'aspera/preview/file_types'
 require 'aspera/persistency_action_once'
 require 'aspera/node'
 require 'aspera/hash_ext'
+require 'aspera/timer_limiter'
+require 'aspera/id_generator'
 require 'date'
 require 'securerandom'
 
@@ -24,7 +26,8 @@ module Aspera
         DEFAULT_PREVIEWS_FOLDER='previews'
         AK_MARKER_FILE='.aspera_access_key'
         LOCAL_STORAGE_PCVL='file:///'
-        private_constant :PREV_GEN_TAG, :PREVIEW_FOLDER_SUFFIX, :PREVIEW_BASENAME, :TMP_DIR_PREFIX, :DEFAULT_PREVIEWS_FOLDER, :LOCAL_STORAGE_PCVL, :AK_MARKER_FILE
+        LOG_LIMITER_SEC=30.0
+        private_constant :PREV_GEN_TAG, :PREVIEW_FOLDER_SUFFIX, :PREVIEW_BASENAME, :TMP_DIR_PREFIX, :DEFAULT_PREVIEWS_FOLDER, :LOCAL_STORAGE_PCVL, :AK_MARKER_FILE, :LOG_LIMITER_SEC
 
         # option_skip_format has special accessors
         attr_accessor :option_previews_folder
@@ -40,6 +43,8 @@ module Aspera
           @preview_formats_to_generate=Aspera::Preview::Generator::PREVIEW_FORMATS.clone
           # options for generation
           @gen_options=Aspera::Preview::Options.new
+          # used to trigger periodic processing
+          @periodic=TimerLimiter.new(LOG_LIMITER_SEC)
           # link CLI options to gen_info attributes
           self.options.set_obj_attr(:skip_format,self,:option_skip_format,[]) # no skip
           self.options.set_obj_attr(:folder_reset_cache,self,:option_folder_reset_cache,:no)
@@ -113,13 +118,14 @@ module Aspera
         end
 
         # old version based on folders
-        def process_trevents(iteration_token)
+        # @param iteration_persistency can be nil
+        def process_trevents(iteration_persistency)
           events_filter={
             'access_key'=>@access_key_self['id'],
             'type'=>'download.ended'
           }
-          # optionally by iteration token
-          events_filter['iteration_token']=iteration_token unless iteration_token.nil?
+          # optionally add iteration token from persistency
+          events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
           begin
             events=@api_node.read('events',events_filter)[:data]
           rescue RestCallError => e
@@ -132,47 +138,63 @@ module Aspera
           end
           return if events.empty?
           events.each do |event|
-            next unless event['data']['direction'].eql?('receive')
-            next unless event['data']['status'].eql?('completed')
-            next unless event['data']['error_code'].eql?(0)
-            next unless event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
-            folder_id=event.dig('data','tags','aspera','node','file_id')
-            folder_id||=event.dig('data','file_id')
-            next if folder_id.nil?
-            folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil
-            next if folder_entry.nil?
-            scan_folder_files(folder_entry)
+            if event['data']['direction'].eql?('receive') and
+            event['data']['status'].eql?('completed') and
+            event['data']['error_code'].eql?(0) and
+            event['data'].dig('tags','aspera',PREV_GEN_TAG).nil?
+              folder_id=event.dig('data','tags','aspera','node','file_id')
+              folder_id||=event.dig('data','file_id')
+              if !folder_id.nil?
+                folder_entry=@api_node.read("files/#{folder_id}")[:data] rescue nil
+                scan_folder_files(folder_entry) unless folder_entry.nil?
+              end
+            end
+            if @periodic.trigger? or event.equal?(events.last)
+              Log.log.info("Processed event #{event['id']}")
+              # save checkpoint to avoid losing processing in case of error
+              if !iteration_persistency.nil?
+                iteration_persistency.data[0]=event['id'].to_s
+                iteration_persistency.save
+              end
+            end
           end
-          return events.last['id'].to_s
         end
 
         # requests recent events on node api and process newly modified folders
-        def process_events(iteration_token)
+        def process_events(iteration_persistency)
           # get new file creation by access key (TODO: what if file already existed?)
           events_filter={
             'access_key'=>@access_key_self['id'],
             'type'=>'file.*'
           }
-          # and optionally by iteration token
-          events_filter['iteration_token']=iteration_token unless iteration_token.nil?
+          # optionally add iteration token from persistency
+          events_filter['iteration_token']=iteration_persistency.data.first unless iteration_persistency.nil?
           events=@api_node.read('events',events_filter)[:data]
           return if events.empty?
           events.each do |event|
             # process only files
-            next unless event.dig('data','type').eql?('file')
-            file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
-            next if file_entry.nil?
-            next unless @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
-            file_entry['parent_file_id']=event['data']['parent_file_id']
-            if event['types'].include?('file.deleted')
-              Log.log.error('TODO'.red)
+            if event.dig('data','type').eql?('file')
+              file_entry=@api_node.read("files/#{event['data']['id']}")[:data] rescue nil
+              if !file_entry.nil? and
+              @option_skip_folders.select{|d|file_entry['path'].start_with?(d)}.empty?
+                file_entry['parent_file_id']=event['data']['parent_file_id']
+                if event['types'].include?('file.deleted')
+                  Log.log.error('TODO'.red)
+                end
+                if event['types'].include?('file.deleted')
+                  generate_preview(file_entry)
+                end
+              end
             end
-            if event['types'].include?('file.deleted')
-              generate_preview(file_entry)
+            if @periodic.trigger? or event.equal?(events.last)
+              Log.log.info("Processing event #{event['id']}")
+              # save checkpoint to avoid losing processing in case of error
+              if !iteration_persistency.nil?
+                iteration_persistency.data[0]=event['id'].to_s
+                iteration_persistency.save
+              end
             end
           end
-          # write new iteration file
-          return events.last['id'].to_s
         end
 
         def do_transfer(direction,folder_id,source_filename,destination='/')
@@ -338,6 +360,7 @@ module Aspera
             entry=entries_to_process.shift
             # process this entry only if it is within the scan_start
             entry_path_with_slash=entry['path']
+            Log.log.info("processing entry #{entry_path_with_slash}") if @periodic.trigger?
             entry_path_with_slash="#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
             if !scan_start.nil? and !scan_start.start_with?(entry_path_with_slash) and !entry_path_with_slash.start_with?(scan_start)
               Log.log.debug("#{entry['path']} folder (skip start)".bg_red)
@@ -441,18 +464,15 @@ module Aspera
             scan_folder_files(folder_info,scan_path)
             return Main.result_status('scan finished')
           when :events,:trevents
-            iteration_data=[]
             iteration_persistency=nil
             if self.options.get_option(:once_only,:mandatory)
               iteration_persistency=PersistencyActionOnce.new(
               manager: @agents[:persistency],
-              data: iteration_data,
-              ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
+              data:    [],
+              id:      IdGenerator.from_list(['preview_iteration',command.to_s,self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)]))
             end
-
-            # call method specified
-            iteration_data[0]=send("process_#{command}",iteration_data[0])
-            iteration_persistency.save unless iteration_persistency.nil?
+            # call processing method specified by command line command
+            send("process_#{command}",iteration_persistency)
             return Main.result_status("#{command} finished")
           when :check
             Aspera::Preview::Utils.check_tools(@skip_types)

data/lib/aspera/cli/transfer_agent.rb

@@ -101,21 +101,29 @@ END_OF_TEMPLATE
             node_config=config.preset_by_name(param_set_name)
           end
           Log.log.debug("node=#{node_config}")
-          raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" if !node_config.is_a?(Hash)
-          # now check there are required parameters
-          sym_config=[:url,:username,:password].inject({}) do |h,param|
-            raise CliBadArgument,"missing parameter [#{param}] in node specification: #{node_config}" if !node_config.has_key?(param.to_s)
-            h[param]=node_config[param.to_s]
-            h
+          raise CliBadArgument,"the node configuration shall be Hash, not #{node_config.class} (#{node_config}), use either @json:<json> or @preset:<parameter set name>" unless node_config.is_a?(Hash)
+          # here, node_config is a Hash
+          node_config=node_config.symbolize_keys
+          # Check mandatory params
+          [:url,:username,:password].each { |k| raise CliBadArgument,"missing parameter [#{k}] in node specification: #{node_config}" unless node_config.has_key?(k) }
+          if node_config[:password].match(/^Bearer /)
+            node_api=Rest.new({
+              base_url: node_config[:url],
+              headers: {
+              'X-Aspera-AccessKey'=>node_config[:username],
+              'Authorization'     =>node_config[:password]}})
+          else
+            node_api=Rest.new({
+              base_url: node_config[:url],
+              auth:     {
+              type:     :basic,
+              username: node_config[:username],
+              password: node_config[:password]
+              }})
           end
-          node_api=Rest.new({
-            :base_url => sym_config[:url],
-            :auth     => {
-            :type     =>:basic,
-            :username => sym_config[:username],
-            :password => sym_config[:password]
-            }})
           new_agent=Fasp::Node.new(node_api)
+          # add root id if it's an access key
+          new_agent.options={root_id: node_config[:root_id]} if node_config.has_key?(:root_id)
         when :aoc
           aoc_config=options.get_option(:transfer_info,:optional)
           if aoc_config.nil?

data/lib/aspera/cli/version.rb

@@ -1,5 +1,5 @@
 module Aspera
   module Cli
-    VERSION = "4.2.2"
+    VERSION = "4.3.0"
   end
 end

data/lib/aspera/command_line_builder.rb

@@ -16,7 +16,6 @@ module Aspera
     # Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
     def self.normalize_description(d)
       d.each do |param_name,options|
-        Log.log.debug("def: #{param_name}")
         raise "Expecting Hash, but have #{options.class} in #{param_name}" unless options.is_a?(Hash)
         #options[:accepted_types]=:bool if options[:cltype].eql?(:envvar) and !options.has_key?(:accepted_types)
         # by default : not mandatory

data/lib/aspera/cos_node.rb

@@ -6,6 +6,7 @@ module Aspera
   class CosNode < Rest
     attr_reader :add_ts
     IBM_CLOUD_TOKEN_URL='https://iam.cloud.ibm.com/identity'
+    TOKEN_FIELD='delegated_refresh_token'
     def initialize(bucket_name,storage_endpoint,instance_id,api_key,auth_url=IBM_CLOUD_TOKEN_URL)
       @auth_url=auth_url
       @api_key=api_key
@@ -32,7 +33,7 @@ module Aspera
       # prepare transfer spec addition
       @add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
         'type'  => 'token',
-        'token' => {'delegated_refresh_token'=>nil}
+        'token' => {TOKEN_FIELD=>nil}
         }}}}}
       generate_token
     end
@@ -45,10 +46,10 @@ module Aspera
         :base_url   => @auth_url,
         :grant      => :delegated_refresh,
         :api_key    => @api_key,
-        :token_field=> 'delegated_refresh_token'
+        :token_field=> TOKEN_FIELD
       })
       # get delagated token to be placed in rest call header and in transfer tags
-      @add_ts['tags']['aspera']['node']['storage_credentials']['token']['delegated_refresh_token']=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
+      @add_ts['tags']['aspera']['node']['storage_credentials']['token'][TOKEN_FIELD]=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
       @params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
     end
   end

data/lib/aspera/fasp/aoc.rb

@@ -11,7 +11,7 @@ module Aspera
         Log.log.warn("Under Development")
         server_node_file = @api_aoc.resolve_node_file(server_home_node_file,server_folder)
         # force node as transfer agent
-        node_api=Fasp::Node.new(@api_aoc.get_node_api(client_node_file[:node_info],AoC::SCOPE_NODE_USER))
+        node_api=Fasp::Node.new(@api_aoc.get_node_api(client_node_file[:node_info],scope: AoC::SCOPE_NODE_USER))
         super(node_api)
         # additional node to node TS info
         @add_ts={