aspera-cli 4.0.0.pre3 → 4.2.1

data/lib/aspera/preview/file_types.rb

@@ -1,9 +1,11 @@
-require 'mimemagic'
-require 'mimemagic/overlay'
+require 'aspera/log'
+require 'singleton'
 
 module Aspera
   module Preview
+    # function conversion_type returns one of the types: CONVERSION_TYPES
     class FileTypes
+      include Singleton
       # values for conversion_type : input format
       CONVERSION_TYPES=[
         :image,
@@ -148,6 +150,7 @@ module Aspera
         'dif' => :office,
         'divx' => :video,
         'dng' => :image,
+        'docx' => :office,
         'dpx' => :image,
         'epdf' => :image,
         'epi' => :image,
@@ -204,6 +207,7 @@ module Aspera
         'pam' => :image,
         'pcd' => :image,
         'pcds' => :image,
+        'pdf' => :pdf,
         'pef' => :image,
         'picon' => :image,
         'pict' => :image,
@@ -261,39 +265,67 @@ module Aspera
         'x3f' => :image,
         'xcf' => :image,
         'xlk' => :office,
+        'xlsx' => :office,
+        'xls' => :office,
         'ycbcr' => :image,
         'ycbcra' => :image,
         'yuv' => :image,
         'zabw' => :office}
 
-      #private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
+      private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
 
-      def self.mime_from_file(filepath)
+      # @attr use_mimemagic [bool] true to use mimemagic to determine real mime type based on file content
+      attr_accessor :use_mimemagic
+
+      def initialize
+        @use_mimemagic=false
+      end
+
+      # use mime magic to find mime type based on file content (magic numbers)
+      def mime_from_file(filepath)
+        # moved here, as mimemagic can cause installation issues
+        require 'mimemagic'
+        require 'mimemagic/version'
+        require 'mimemagic/overlay' if MimeMagic::VERSION.start_with?('0.3.')
+        # check magic number inside file (empty string if not found)
         detected_mime=MimeMagic.by_magic(File.open(filepath)).to_s
-        detected_mime=MimeMagic.by_path(filepath).to_s if detected_mime.empty?
+        # check extension only
+        if !SUPPORTED_MIME_TYPES.has_key?(detected_mime)
+          Log.log.debug("no conversion for #{detected_mime}, trying extension")
+          detected_mime=MimeMagic.by_extension(File.extname(filepath)).to_s
+        end
         detected_mime=nil if detected_mime.empty?
+        Log.log.debug("mimemagic: #{detected_mime.class.name} [#{detected_mime}]")
         return detected_mime
       end
 
-      def self.conversion_type(filepath,mimetype,try_local_mime)
-        detected_mime=nil
-        if try_local_mime
+      # return file type, one of enum CONVERSION_TYPES
+      # @param filepath [String] full path to file
+      # @param mimetype [String] provided by node api
+      def conversion_type(filepath,mimetype)
+        Log.log.debug("conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})")
+        # 1- get type from provided mime type, using local mapping
+        conv_type=SUPPORTED_MIME_TYPES[mimetype] if ! mimetype.nil?
+        # 2- else, from computed mime type (if available)
+        if conv_type.nil? and @use_mimemagic
           detected_mime=mime_from_file(filepath)
-          if mimetype.eql?(detected_mime)
-            Log.log.debug("matching mime type per magic number")
-          else
-            # note: detected can be nil
-            Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
+          if ! detected_mime.nil?
+            conv_type=SUPPORTED_MIME_TYPES[detected_mime]
+            if ! mimetype.nil?
+              if mimetype.eql?(detected_mime)
+                Log.log.debug("matching mime type per magic number")
+              else
+                # note: detected can be nil
+                Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
+              end
+            end
           end
         end
-        # 1- get type from provided mime type
-        result=FileTypes::SUPPORTED_MIME_TYPES[mimetype] unless mimetype.nil?
-        # 2- else, from computed mime type
-        result||=FileTypes::SUPPORTED_MIME_TYPES[detected_mime] unless detected_mime.nil?
-        extension = File.extname(filepath).downcase.gsub(/^\./,'')
-        # 3- else, from local extensions
-        result||=FileTypes::SUPPORTED_EXTENSIONS[extension]
-        return result
+        # 3- else, from extensions, using local mapping
+        extension = File.extname(filepath.downcase)[1..-1]
+        conv_type=SUPPORTED_EXTENSIONS[extension] if conv_type.nil?
+        Log.log.debug("conversion_type(#{extension}): #{conv_type.class.name} [#{conv_type}]")
+        return conv_type
       end
     end
   end

data/lib/aspera/preview/generator.rb

@@ -15,7 +15,7 @@ module Aspera
 
       # @param src source file path
       # @param dst destination file path
-      # @param api_mime_type optional mime type as provided by node api
+      # @param api_mime_type optional mime type as provided by node api (or nil)
       # node API mime types are from: http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
       # supported preview type is one of Preview::PREVIEW_FORMATS
       # the resulting preview file type is taken from destination file extension.
@@ -25,14 +25,14 @@ module Aspera
       #   -> preview_format is one of Generator::PREVIEW_FORMATS
       # the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
       #  -> conversion method is one of Generator::VIDEO_CONVERSION_METHODS
-      def initialize(options,src,dst,main_temp_dir,api_mime_type=nil,try_local_mime=true)
+      def initialize(options,src,dst,main_temp_dir,api_mime_type)
         @options=options
         @source_file_path=src
         @destination_file_path=dst
         @temp_folder=File.join(main_temp_dir,@source_file_path.split('/').last.gsub(/\s/, '_').gsub(/\W/, ''))
         # extract preview format from extension of target file
         @preview_format_symb=File.extname(@destination_file_path).gsub(/^\./,'').to_sym
-        @conversion_type=FileTypes.conversion_type(@source_file_path,api_mime_type,try_local_mime)
+        @conversion_type=FileTypes.instance.conversion_type(@source_file_path,api_mime_type)
       end
 
       # name of processing method in this object

data/lib/aspera/rest.rb

@@ -7,6 +7,7 @@ require 'net/http'
 require 'net/https'
 require 'json'
 require 'base64'
+require 'cgi'
 require 'ruby-progressbar'
 
 # add cancel method to http
@@ -33,6 +34,7 @@ module Aspera
     # true if https ignore certificate
     @@insecure=false
     @@user_agent='Ruby'
+    @@download_partial_suffix='.http_partial'
 
     public
     def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
@@ -98,9 +100,10 @@ module Aspera
     public
 
     attr_reader :params
+    attr_reader :oauth
 
-    # @param a_rest_params default call parameters and authentication (:auth) :
-    # :type (:basic, :oauth2, :url)
+    # @param a_rest_params default call parameters (merged at call) and (+) authentication (:auth) :
+    # :type (:basic, :oauth2, :url, :none)
     # :username   [:basic]
     # :password   [:basic]
     # :url_creds  [:url]
@@ -117,16 +120,6 @@ module Aspera
       # default is no auth
       @params[:auth]||={:type=>:none}
       @params[:not_auth_codes]||=['401']
-      # translate old auth parameters, remove prefix, place in auth
-      [:auth,:basic,:oauth].each do |p_sym|
-        p_str=p_sym.to_s+'_'
-        @params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
-          name=k_sym.to_s[p_str.length..-1]
-          name='grant' if k_sym.eql?(:oauth_type)
-          @params[:auth][name.to_sym]=@params[k_sym]
-          @params.delete(k_sym)
-        end
-      end
       @oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
       Log.dump('REST params(2)',@params)
     end
@@ -153,6 +146,7 @@ module Aspera
       Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
       call_data[:headers]||={}
       call_data[:headers]['User-Agent'] ||= @@user_agent
+      # defaults from @params are overriden by call dataz
       call_data=@params.deep_merge(call_data)
       case call_data[:auth][:type]
       when :none
@@ -206,10 +200,13 @@ module Aspera
 
       Log.log.debug("call_data = #{call_data}")
       result={:http=>nil}
+      # start a block to be able to retry the actual HTTP request
       begin
         # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
         oauth_tries ||= 2
+        tries_remain_redirect||=4
         Log.log.debug("send request")
+        # make http request (pipelined)
         http_session.request(req) do |response|
           result[:http] = response
           if call_data.has_key?(:save_to_file)
@@ -223,12 +220,12 @@ module Aspera
             target_file=call_data[:save_to_file]
             # override user's path to path in header
             if !response['Content-Disposition'].nil? and m=response['Content-Disposition'].match(/filename="([^"]+)"/)
-              target_file=m[1]
+              target_file=File.join(File.dirname(target_file),m[1])
             end
             # download with temp filename
-            target_file_tmp=target_file+'.http.partial'
+            target_file_tmp="#{target_file}#{@@download_partial_suffix}"
             Log.log.debug("saving to: #{target_file}")
-            File.open(target_file_tmp, "wb") do |file|
+            File.open(target_file_tmp, 'wb') do |file|
               result[:http].read_body do |fragment|
                 file.write(fragment)
                 new_process=progress.progress+fragment.length
@@ -239,7 +236,7 @@ module Aspera
             # rename at the end
             File.rename(target_file_tmp, target_file)
             progress=nil
-          end
+          end # save_to_file
         end
         # sometimes there is a ITF8 char (e.g. (c) )
         result[:http].body.force_encoding("UTF-8") if result[:http].body.is_a?(String)
@@ -267,10 +264,24 @@ module Aspera
           end
           Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
           retry unless (oauth_tries -= 1).zero?
-        end # if
+        end # if oauth
+        # moved ?
+        if e.response.is_a?(Net::HTTPRedirection)
+          if tries_remain_redirect > 0
+            tries_remain_redirect-=1
+            Log.log.info("URL is moved: #{e.response['location']}")
+            raise e
+            # TODO: rebuild request with new location
+            #retry
+          else
+            raise "too many redirect"
+          end
+        else
+          raise e
+        end
         # raise exception if could not retry and not return error in result
         raise e unless call_data[:return_error]
-      end
+      end # begin request
       Log.log.debug("result=#{result}")
       return result
 

data/lib/aspera/secrets.rb

@@ -0,0 +1,20 @@
+module Aspera
+  # Manage secrets in CLI using secure way (encryption, wallet, etc...)
+  class Secrets
+    attr_accessor :default_secret,:all_secrets
+    def initialize()
+      @default_secret=nil
+      @all_secrets={}
+    end
+
+    def get_secret(id=nil,mandatory=true)
+      secret=@default_secret || @all_secrets[id]
+      raise "please provide secret for #{id}" if secret.nil? and mandatory
+      return secret
+    end
+
+    def get_secrets
+      return @all_secrets
+    end
+  end
+end

data/lib/aspera/temp_file_manager.rb

@@ -6,6 +6,11 @@ module Aspera
   # create a temp file name for a given folder
   # files can be deleted on process exit by calling cleanup
   class TempFileManager
+    SEC_IN_DAY=86400
+    # assume no transfer last longer than this
+    # (garbage collect file list which were not deleted after transfer)
+    FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
+    private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC
     include Singleton
     def initialize
       @created_files=[]
@@ -33,5 +38,19 @@ module Aspera
       username = Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user' rescue 'unknown_user'
       return new_file_path_in_folder(Etc.systmpdir,base_name+'_'+username+'_')
     end
+
+    def cleanup_expired(temp_folder)
+      # garbage collect undeleted files
+      Dir.entries(temp_folder).each do |name|
+        file_path=File.join(temp_folder,name)
+        age_sec=(Time.now - File.stat(file_path).mtime).to_i
+        # check age of file, delete too old
+        if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
+          Log.log.debug("garbage collecting #{name}")
+          File.delete(file_path)
+        end
+      end
+
+    end
   end
 end

data/lib/aspera/web_auth.rb

@@ -0,0 +1,105 @@
+require 'webrick'
+require 'webrick/https'
+require 'thread'
+
+module Aspera
+  class WebAuth
+    # server for auth page
+    class FxGwServlet < WEBrick::HTTPServlet::AbstractServlet
+      def initialize(server,info) # additional args get here
+        @shared=info
+      end
+
+      def do_GET (request, response)
+        if ! request.path.eql?(@shared[:expected_path])
+          response.status=400
+          return
+        end
+        @shared[:mutex].synchronize do
+          @shared[:query]=request.query
+          @shared[:cond].signal
+        end
+        response.status=200
+        response.content_type = 'text/html'
+        response.body='<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>'
+      end
+    end # FxGwServlet
+
+    # generates and adds self signed cert to provided webrick options
+    def fill_self_signed_cert(options)
+      key = OpenSSL::PKey::RSA.new(4096)
+      cert = OpenSSL::X509::Certificate.new
+      cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/C=FR/O=Test/OU=Test/CN=Test')
+      cert.not_before = Time.now
+      cert.not_after = Time.now + 365 * 24 * 60 * 60
+      cert.public_key = key.public_key
+      cert.serial = 0x0
+      cert.version = 2
+      ef = OpenSSL::X509::ExtensionFactory.new
+      ef.issuer_certificate = cert
+      ef.subject_certificate = cert
+      cert.extensions = [
+        ef.create_extension('basicConstraints','CA:TRUE', true),
+        ef.create_extension('subjectKeyIdentifier', 'hash'),
+        # ef.create_extension('keyUsage', 'cRLSign,keyCertSign', true),
+      ]
+      cert.add_extension(ef.create_extension('authorityKeyIdentifier','keyid:always,issuer:always'))
+      cert.sign(key, OpenSSL::Digest::SHA256.new)
+      options[:SSLPrivateKey]  = key
+      options[:SSLCertificate] = cert
+    end
+
+    def initialize(endpoint_url)
+      uri=URI.parse(endpoint_url)
+      webrick_options = {
+        :app                => WebAuth,
+        :Port               => uri.port,
+        :Logger             => Log.log
+      }
+      uri_path=uri.path.empty? ? '/' : uri.path
+      case uri.scheme
+      when 'http'
+        Log.log.debug('HTTP mode')
+      when 'https'
+        webrick_options[:SSLEnable]=true
+        webrick_options[:SSLVerifyClient]=OpenSSL::SSL::VERIFY_NONE
+        case 0
+        when 0
+          # generate self signed cert
+          fill_self_signed_cert(webrick_options)
+        when 1
+          # short
+          webrick_options[:SSLCertName]    = [ [ 'CN',WEBrick::Utils::getservername ] ]
+          Log.log.error(">>>#{webrick_options[:SSLCertName]}")
+        when 2
+          # good cert
+          webrick_options[:SSLPrivateKey] =OpenSSL::PKey::RSA.new(File.read('/Users/laurent/workspace/Tools/certificate/myserver.key'))
+          webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read('/Users/laurent/workspace/Tools/certificate/myserver.crt'))
+        end
+      end
+      # parameters for servlet
+      @shared_info={
+        expected_path: uri_path,
+        mutex: Mutex.new,
+        cond: ConditionVariable.new
+      }
+      @server = WEBrick::HTTPServer.new(webrick_options)
+      @server.mount(uri_path, FxGwServlet, @shared_info) # additional args provided to constructor
+      Thread.new { @server.start }
+    end
+
+    # wait for request on web server
+    # @return Hash the query
+    def get_request
+      Log.log.debug('get_request')
+      # called only once
+      raise "error, called twice ?" if @server.nil?
+      @shared_info[:mutex].synchronize do
+        @shared_info[:cond].wait(@shared_info[:mutex])
+      end
+      @server.shutdown
+      @server=nil
+      return @shared_info[:query]
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aspera-cli
 version: !ruby/object:Gem::Version
-  version: 4.0.0.pre3
+  version: 4.2.1
 platform: ruby
 authors:
 - Laurent Martin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-18 00:00:00.000000000 Z
+date: 2021-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: xml-simple
@@ -58,28 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
-- !ruby/object:Gem::Dependency
-  name: mimemagic
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: execjs
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +122,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: websocket
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: websocket-client-simple
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -184,12 +198,14 @@ email:
 executables:
 - ascli
 - asession
+- dascli
 extensions: []
 extra_rdoc_files: []
 files:
 - README.md
 - bin/ascli
 - bin/asession
+- bin/dascli
 - docs/Makefile
 - docs/README.erb.md
 - docs/README.md
@@ -197,6 +213,7 @@ files:
 - docs/test_env.conf
 - docs/transfer_spec.html
 - examples/aoc.rb
+- examples/faspex4.rb
 - examples/proxy.pac
 - examples/transfer.rb
 - lib/aspera/aoc.rb
@@ -277,10 +294,12 @@ files:
 - lib/aspera/rest_call_error.rb
 - lib/aspera/rest_error_analyzer.rb
 - lib/aspera/rest_errors_aspera.rb
+- lib/aspera/secrets.rb
 - lib/aspera/ssh.rb
 - lib/aspera/sync.rb
 - lib/aspera/temp_file_manager.rb
 - lib/aspera/uri_reader.rb
+- lib/aspera/web_auth.rb
 homepage: https://github.com/IBM/aspera-cli
 licenses:
 - Apache-2.0
@@ -290,6 +309,7 @@ metadata:
   source_code_uri: https://github.com/IBM/aspera-cli
   changelog_uri: https://github.com/IBM/aspera-cli
   rubygems_uri: https://rubygems.org/gems/aspera-cli
+  documentation_uri: https://www.rubydoc.info/gems/aspera-cli
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -301,12 +321,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements:
-- IBM Aspera ascp installed for the user
-rubygems_version: 3.0.3
+- Read the manual for any requirement
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: 'Execute actions using command line on IBM Aspera Server products: Aspera