aspera-cli 4.0.0 → 4.1.0

data/lib/aspera/fasp/installation.rb

@@ -176,10 +176,34 @@ module Aspera
         return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
       end
 
-      def install_sdk
+      # download aspera SDK or use local file
+      # extract only ascp binary for current system architecture
+      def install_sdk(sdk_url)
         require 'zip'
         sdk_zip_path=File.join(Dir.tmpdir,'sdk.zip')
-        Aspera::Rest.new(base_url: SDK_URL).call(operation: 'GET',save_to_file: sdk_zip_path)
+        if sdk_url.start_with?('file:')
+          # require specific file scheme: the path part is "relative", or absolute if there are 4 slash
+          raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
+          sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
+        else
+          redirect_remain=2
+          begin
+            Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
+          rescue Aspera::RestCallError => e
+            if e.response.is_a?(Net::HTTPRedirection)
+              if redirect_remain > 0
+                redirect_remain-=1
+                sdk_url=e.response['location']
+                retry
+              else
+                raise "too meny redirect"
+              end
+            else
+              raise e
+            end
+          end
+        end
+        # SDK is organized by architecture
         filter="/#{Environment.architecture}/"
         ascp_path=nil
         # first ensure license file is here so that ascp invokation for version works
@@ -187,6 +211,7 @@ module Aspera
         self.path(:aspera_conf)
         Zip::File.open(sdk_zip_path) do |zip_file|
           zip_file.each do |entry|
+            # get only specified arch, but not folder, only files
             if entry.name.include?(filter) and !entry.name.end_with?('/')
               archive_file=File.join(folder_path,File.basename(entry.name))
               File.open(archive_file, 'wb') do |output_stream|
@@ -220,9 +245,8 @@ module Aspera
       PRODUCT_INFO='product-info.mf'
       # policy for product selection
       FIRST_FOUND='FIRST'
-      SDK_URL='https://eudemo.asperademo.com/aspera/faspex/sdk.zip'
 
-      private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO,:SDK_URL
+      private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO
 
       # get some specific folder from specific applications: Connect or CLI
       def get_product_folders(name)

data/lib/aspera/fasp/local.rb

@@ -232,23 +232,26 @@ module Aspera
             end # case
           end # loop (process mgt port lines)
           # check that last status was received before process exit
-          raise "INTERNAL: nil last status" if last_status_event.nil?
-          case last_status_event['Type']
-          when 'DONE'
-            # return method (or just don't do anything)
-            return
-          when 'ERROR'
-            Log.log.error("code: #{last_status_event['Code']}")
-            if last_status_event['Description']  =~ /bearer token/i
-              Log.log.error("need to regenerate token".red)
-              if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
-                # regenerate token here, expired, or error on it
-                env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
+          if last_status_event.nil?
+            Log.log.warn("no status read from ascp mgt port")
+          else
+            case last_status_event['Type']
+            when 'DONE'
+              # return method (or just don't do anything)
+              return
+            when 'ERROR'
+              Log.log.error("code: #{last_status_event['Code']}")
+              if last_status_event['Description']  =~ /bearer token/i
+                Log.log.error("need to regenerate token".red)
+                if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
+                  # regenerate token here, expired, or error on it
+                  env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
+                end
               end
+              raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
+            else
+              raise "unexpected last event type: #{last_status_event['Type']}"
             end
-            raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
-          else
-            raise "unexpected last event type: #{last_status_event['Type']}"
           end
         rescue SystemCallError => e
           # Process.spawn
@@ -262,8 +265,12 @@ module Aspera
           unless ascp_pid.nil?
             # "wait" for process to avoid zombie
             Process.wait(ascp_pid)
+            status=$?
             ascp_pid=nil
             session.delete(:io)
+            if !status.success?
+              raise Fasp::Error.new("ascp failed with code #{status.exitstatus}")
+            end
           end
         end # begin-ensure
       end # start_transfer_with_args_env
@@ -301,8 +308,9 @@ module Aspera
         @jobs={}
         # mutex protects global data accessed by threads
         @mutex=Mutex.new
-        @resume_policy=ResumePolicy.new(agent_options)
         @enable_wss = agent_options[:wss] || false
+        agent_options.delete(:wss)
+        @resume_policy=ResumePolicy.new(agent_options)
       end
 
       # transfer thread entry

data/lib/aspera/fasp/manager.rb

@@ -45,6 +45,18 @@ module Aspera
         end
       end # notify_listeners
 
+      def notify_begin(id,size)
+        notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'NOTIFICATION','PreTransferBytes'=>size})
+      end
+
+      def notify_progress(id,size)
+        notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'STATS','Bytescont'=>size})
+      end
+
+      def notify_end(id)
+        notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'DONE'})
+      end
+
       public
       LISTENER_SESSION_ID_B='ListenerSessionId'
       LISTENER_SESSION_ID_S='listener_session_id'

data/lib/aspera/fasp/node.rb

@@ -55,7 +55,7 @@ module Aspera
           trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
           case trdata['status']
           when 'completed'
-            notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'DONE'})
+            notify_end(@transfer_id)
             break
           when 'waiting','partially_completed','unknown','waiting(read error)'
             if spinner.nil?
@@ -69,10 +69,10 @@ module Aspera
             #puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
             if !started and trdata['precalc'].is_a?(Hash) and
             trdata['precalc']['status'].eql?('ready')
-              notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'NOTIFICATION','PreTransferBytes'=>trdata['precalc']['bytes_expected']})
+              notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
               started=true
             else
-              notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'STATS','Bytescont'=>trdata['bytes_transferred']})
+              notify_progress(@transfer_id,trdata['bytes_transferred'])
             end
           else
             Log.log.warn("trdata -> #{trdata}")
@@ -81,7 +81,7 @@ module Aspera
           sleep 1
         end
         #TODO get status of sessions
-        return [] 
+        return []
       end
     end
   end

data/lib/aspera/fasp/parameters.rb

@@ -16,10 +16,6 @@ module Aspera
       # because of garbage collection takes any file there
       # this could be refined, as , for instance, on macos, temp folder is already user specific
       @@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
-      SEC_IN_DAY=86400
-      # assume no transfer last longer than this
-      # (garbage collect file list which were not deleted after transfer)
-      FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
       PARAM_DEFINITION={
         # parameters with env vars
         'remote_password'         => { :type => :envvar, :variable=>'ASPERA_SCP_PASS'},
@@ -100,7 +96,7 @@ module Aspera
         'wss_port'                => { :type => :defer, :accepted_types=>Integer},
       }
 
-      private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC,:PARAM_DEFINITION
+      private_constant :PARAM_DEFINITION
 
       def initialize(job_spec,options)
         @job_spec=job_spec
@@ -207,18 +203,9 @@ module Aspera
       # temp file list files are created here
       def self.file_list_folder=(v)
         @@file_list_folder=v
-        unless @@file_list_folder.nil?
+        if !@@file_list_folder.nil?
           FileUtils.mkdir_p(@@file_list_folder)
-          # garbage collect undeleted files
-          Dir.entries(@@file_list_folder).each do |name|
-            file_path=File.join(@@file_list_folder,name)
-            age_sec=(Time.now - File.stat(file_path).mtime).to_i
-            # check age of file, delete too old
-            if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
-              Log.log.debug("garbage collecting #{name}")
-              File.delete(file_path)
-            end
-          end
+          TempFileManager.instance.cleanup_expired(@@file_list_folder)
         end
       end
 

data/lib/aspera/log.rb

@@ -14,7 +14,7 @@ module Aspera
     attr_reader :logger
     attr_reader :logger_type
     # levels are :debug,:info,:warn,:error,fatal,:unknown
-    def self.levels; Logger::Severity.constants.map{|c| c.downcase.to_sym};end
+    def self.levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a)<=>Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
 
     # where logs are sent to
     def self.logtypes; [:stderr,:stdout,:syslog];end

data/lib/aspera/node.rb

@@ -1,14 +1,61 @@
+require 'aspera/rest'
+require 'aspera/log'
 require 'zlib'
 require 'base64'
+
 module Aspera
   # Provides additional functions using node API.
   class Node < Rest
+    # permissions
+    ACCESS_LEVELS=['delete','list','mkdir','preview','read','rename','write']
+
+    # for information only
     def self.decode_bearer_token(token)
       return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)
     end
+
     def initialize(rest_params)
       super(rest_params)
-      # specifics here
+    end
+
+    # recursively crawl in a folder.
+    # subfolders a processed if the processing method returns true
+    # @param processor must provide a method to process each entry
+    # @param opt options
+    # - top_file_id file id to start at (default = access key root file id)
+    # - top_file_path path of top folder (default = /)
+    # - method processing method (default= process_entry)
+    def crawl(processor,opt={})
+      Log.log.debug("crawl1 #{opt}")
+      # not possible with bearer token
+      opt[:top_file_id] ||= read('access_keys/self')[:data]['root_file_id']
+      opt[:top_file_path] ||= '/'
+      opt[:method] ||= :process_entry
+      raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
+      Log.log.debug("crawl #{opt}")
+      #top_info=read("files/#{opt[:top_file_id]}")[:data]
+      folders_to_explore=[{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
+      Log.dump(:folders_to_explore,folders_to_explore)
+      while !folders_to_explore.empty? do
+        current_item = folders_to_explore.shift
+        Log.log.debug("searching #{current_item[:relpath]}".bg_green)
+        # get folder content
+        folder_contents = begin
+          read("files/#{current_item[:id]}/files")[:data]
+        rescue => e
+          Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
+          []
+        end
+        Log.dump(:folder_contents,folder_contents)
+        folder_contents.each do |entry|
+          relative_path=File.join(current_item[:relpath],entry['name'])
+          Log.log.debug("looking #{relative_path}".bg_green)
+          # entry type is file, folder or link
+          if processor.send(opt[:method],entry,relative_path) and entry['type'].eql?('folder')
+            folders_to_explore.push({:id=>entry['id'],:relpath=>relative_path})
+          end
+        end
+      end
     end
   end
 end

data/lib/aspera/oauth.rb

@@ -16,7 +16,10 @@ module Aspera
     # one hour validity (TODO: configurable?)
     JWT_EXPIRY_OFFSET=3600
     PERSIST_CATEGORY_TOKEN='token'
-    private_constant :JWT_NOTBEFORE_OFFSET,:JWT_EXPIRY_OFFSET,:PERSIST_CATEGORY_TOKEN
+    # tokens older than 30 minutes will be discarded
+    TOKEN_EXPIRY_SECONDS=1800
+    THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
+    private_constant :JWT_NOTBEFORE_OFFSET,:JWT_EXPIRY_OFFSET,:PERSIST_CATEGORY_TOKEN,:TOKEN_EXPIRY_SECONDS,:THANK_YOU_HTML
     class << self
       # OAuth methods supported
       def auth_types
@@ -28,12 +31,18 @@ module Aspera
       end
 
       def persist_mgr
-        raise "set persistency manager first" if @persist.nil?
+        if @persist.nil?
+          Log.log.warn('Not using persistency (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)')
+          # create NULL persistency class
+          @persist=Class.new do
+            def get(x);nil;end;def delete(x);nil;end;def put(x,y);nil;end;def garbage_collect(x,y);nil;end
+          end.new
+        end
         return @persist
       end
 
       def flush_tokens
-        persist_mgr.flush_by_prefix(PERSIST_CATEGORY_TOKEN)
+        persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
       end
     end
 
@@ -79,10 +88,10 @@ module Aspera
         raise "redirect_uri must have a port" if uri.port.nil?
         # we could check that host is localhost or local address
       end
+      # cleanup expired tokens
+      self.class.persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_EXPIRY_SECONDS)
     end
 
-    THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
-
     # open the login page, wait for code and check_code, then return code
     def goto_page_and_get_code(login_page_url,check_code)
       request_params=self.class.goto_page_and_get_request(@params[:redirect_uri],login_page_url)
@@ -117,6 +126,9 @@ module Aspera
 
     public
 
+    # used to change parameter, such as scope
+    attr_reader :params
+
     # @param options : :scope and :refresh
     def get_authorization(options={})
       # api scope can be overriden to get auth for other scope
@@ -184,14 +196,13 @@ module Aspera
         when :web
           # AoC Web based Auth
           check_code=SecureRandom.uuid
-          login_page_url=Rest.build_uri(
-          "#{@params[:base_url]}/#{@params[:path_authorize]}",
-          p_client_id_and_scope.merge({
+          auth_params=p_client_id_and_scope.merge({
             :response_type => 'code',
             :redirect_uri  => @params[:redirect_uri],
-            :client_secret => @params[:client_secret],
             :state         => check_code
-          }))
+          })
+          auth_params[:client_secret]=@params[:client_secret] if @params.has_key?(:client_secret)
+          login_page_url=Rest.build_uri("#{@params[:base_url]}/#{@params[:path_authorize]}",auth_params)
           # here, we need a human to authorize on a web page
           code=goto_page_and_get_code(login_page_url,check_code)
           # exchange code for token
@@ -233,8 +244,10 @@ module Aspera
           }))
         when :url_token
           # AoC Public Link
+          params={:url_token=>@params[:url_token]}
+          params[:password]=@params[:password] if @params.has_key?(:password)
           resp=create_token_advanced({
-            :json_params => {:url_token=>@params[:url_token]},
+            :json_params => params,
             :url_params  => p_scope.merge({
             :grant_type    => 'url_token'
             })})

data/lib/aspera/persistency_folder.rb

@@ -55,12 +55,17 @@ module Aspera
       @cache.delete(object_id)
     end
 
-    def flush_by_prefix(persist_category)
-      persist_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
-      persist_files.each do |filepath|
+    def garbage_collect(persist_category,max_age_seconds=nil)
+      garbage_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
+      if !max_age_seconds.nil?
+        current_time = Time.now
+        garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
+      end
+      garbage_files.each do |filepath|
         File.delete(filepath)
+        Log.log.debug("Deleted expired: #{filepath}")
       end
-      return persist_files
+      return garbage_files
     end
 
     private

data/lib/aspera/preview/file_types.rb

@@ -1,9 +1,11 @@
-require 'mimemagic'
-require 'mimemagic/overlay'
+require 'aspera/log'
+require 'singleton'
 
 module Aspera
   module Preview
+    # function conversion_type returns one of the types: CONVERSION_TYPES
     class FileTypes
+      include Singleton
       # values for conversion_type : input format
       CONVERSION_TYPES=[
         :image,
@@ -148,6 +150,7 @@ module Aspera
         'dif' => :office,
         'divx' => :video,
         'dng' => :image,
+        'docx' => :office,
         'dpx' => :image,
         'epdf' => :image,
         'epi' => :image,
@@ -204,6 +207,7 @@ module Aspera
         'pam' => :image,
         'pcd' => :image,
         'pcds' => :image,
+        'pdf' => :pdf,
         'pef' => :image,
         'picon' => :image,
         'pict' => :image,
@@ -261,39 +265,67 @@ module Aspera
         'x3f' => :image,
         'xcf' => :image,
         'xlk' => :office,
+        'xlsx' => :office,
+        'xls' => :office,
         'ycbcr' => :image,
         'ycbcra' => :image,
         'yuv' => :image,
         'zabw' => :office}
 
-      #private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
+      private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
 
-      def self.mime_from_file(filepath)
+      # @attr use_mimemagic [bool] true to use mimemagic to determine real mime type based on file content
+      attr_accessor :use_mimemagic
+
+      def initialize
+        @use_mimemagic=false
+      end
+
+      # use mime magic to find mime type based on file content (magic numbers)
+      def mime_from_file(filepath)
+        # moved here, as mimemagic can cause installation issues
+        require 'mimemagic'
+        require 'mimemagic/version'
+        require 'mimemagic/overlay' if MimeMagic::VERSION.start_with?('0.3.')
+        # check magic number inside file (empty string if not found)
         detected_mime=MimeMagic.by_magic(File.open(filepath)).to_s
-        detected_mime=MimeMagic.by_path(filepath).to_s if detected_mime.empty?
+        # check extension only
+        if !SUPPORTED_MIME_TYPES.has_key?(detected_mime)
+          Log.log.debug("no conversion for #{detected_mime}, trying extension")
+          detected_mime=MimeMagic.by_extension(File.extname(filepath)).to_s
+        end
         detected_mime=nil if detected_mime.empty?
+        Log.log.debug("mimemagic: #{detected_mime.class.name} [#{detected_mime}]")
         return detected_mime
       end
 
-      def self.conversion_type(filepath,mimetype,try_local_mime)
-        detected_mime=nil
-        if try_local_mime
+      # return file type, one of enum CONVERSION_TYPES
+      # @param filepath [String] full path to file
+      # @param mimetype [String] provided by node api
+      def conversion_type(filepath,mimetype)
+        Log.log.debug("conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})")
+        # 1- get type from provided mime type, using local mapping
+        conv_type=SUPPORTED_MIME_TYPES[mimetype] if ! mimetype.nil?
+        # 2- else, from computed mime type (if available)
+        if conv_type.nil? and @use_mimemagic
           detected_mime=mime_from_file(filepath)
-          if mimetype.eql?(detected_mime)
-            Log.log.debug("matching mime type per magic number")
-          else
-            # note: detected can be nil
-            Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
+          if ! detected_mime.nil?
+            conv_type=SUPPORTED_MIME_TYPES[detected_mime]
+            if ! mimetype.nil?
+              if mimetype.eql?(detected_mime)
+                Log.log.debug("matching mime type per magic number")
+              else
+                # note: detected can be nil
+                Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
+              end
+            end
           end
         end
-        # 1- get type from provided mime type
-        result=FileTypes::SUPPORTED_MIME_TYPES[mimetype] unless mimetype.nil?
-        # 2- else, from computed mime type
-        result||=FileTypes::SUPPORTED_MIME_TYPES[detected_mime] unless detected_mime.nil?
-        extension = File.extname(filepath).downcase.gsub(/^\./,'')
-        # 3- else, from local extensions
-        result||=FileTypes::SUPPORTED_EXTENSIONS[extension]
-        return result
+        # 3- else, from extensions, using local mapping
+        extension = File.extname(filepath.downcase)[1..-1]
+        conv_type=SUPPORTED_EXTENSIONS[extension] if conv_type.nil?
+        Log.log.debug("conversion_type(#{extension}): #{conv_type.class.name} [#{conv_type}]")
+        return conv_type
       end
     end
   end