aspera-cli 4.0.0.pre2 → 4.2.0

data/lib/aspera/preview/generator.rb

@@ -15,7 +15,7 @@ module Aspera
 
       # @param src source file path
       # @param dst destination file path
-      # @param api_mime_type optional mime type as provided by node api
+      # @param api_mime_type optional mime type as provided by node api (or nil)
       # node API mime types are from: http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
       # supported preview type is one of Preview::PREVIEW_FORMATS
       # the resulting preview file type is taken from destination file extension.
@@ -25,14 +25,14 @@ module Aspera
       #   -> preview_format is one of Generator::PREVIEW_FORMATS
       # the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
       #  -> conversion method is one of Generator::VIDEO_CONVERSION_METHODS
-      def initialize(options,src,dst,main_temp_dir,api_mime_type=nil,try_local_mime=true)
+      def initialize(options,src,dst,main_temp_dir,api_mime_type)
         @options=options
         @source_file_path=src
         @destination_file_path=dst
         @temp_folder=File.join(main_temp_dir,@source_file_path.split('/').last.gsub(/\s/, '_').gsub(/\W/, ''))
         # extract preview format from extension of target file
         @preview_format_symb=File.extname(@destination_file_path).gsub(/^\./,'').to_sym
-        @conversion_type=FileTypes.conversion_type(@source_file_path,api_mime_type,try_local_mime)
+        @conversion_type=FileTypes.instance.conversion_type(@source_file_path,api_mime_type)
       end
 
       # name of processing method in this object

data/lib/aspera/rest.rb

@@ -7,6 +7,7 @@ require 'net/http'
 require 'net/https'
 require 'json'
 require 'base64'
+require 'cgi'
 require 'ruby-progressbar'
 
 # add cancel method to http
@@ -33,6 +34,7 @@ module Aspera
     # true if https ignore certificate
     @@insecure=false
     @@user_agent='Ruby'
+    @@download_partial_suffix='.http_partial'
 
     public
     def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
@@ -98,9 +100,10 @@ module Aspera
     public
 
     attr_reader :params
+    attr_reader :oauth
 
-    # @param a_rest_params default call parameters and authentication (:auth) :
-    # :type (:basic, :oauth2, :url)
+    # @param a_rest_params default call parameters (merged at call) and (+) authentication (:auth) :
+    # :type (:basic, :oauth2, :url, :none)
     # :username   [:basic]
     # :password   [:basic]
     # :url_creds  [:url]
@@ -117,16 +120,6 @@ module Aspera
       # default is no auth
       @params[:auth]||={:type=>:none}
       @params[:not_auth_codes]||=['401']
-      # translate old auth parameters, remove prefix, place in auth
-      [:auth,:basic,:oauth].each do |p_sym|
-        p_str=p_sym.to_s+'_'
-        @params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
-          name=k_sym.to_s[p_str.length..-1]
-          name='grant' if k_sym.eql?(:oauth_type)
-          @params[:auth][name.to_sym]=@params[k_sym]
-          @params.delete(k_sym)
-        end
-      end
       @oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
       Log.dump('REST params(2)',@params)
     end
@@ -153,6 +146,7 @@ module Aspera
       Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
       call_data[:headers]||={}
       call_data[:headers]['User-Agent'] ||= @@user_agent
+      # defaults from @params are overriden by call dataz
       call_data=@params.deep_merge(call_data)
       case call_data[:auth][:type]
       when :none
@@ -206,10 +200,13 @@ module Aspera
 
       Log.log.debug("call_data = #{call_data}")
       result={:http=>nil}
+      # start a block to be able to retry the actual HTTP request
       begin
         # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
         oauth_tries ||= 2
+        tries_remain_redirect||=4
         Log.log.debug("send request")
+        # make http request (pipelined)
         http_session.request(req) do |response|
           result[:http] = response
           if call_data.has_key?(:save_to_file)
@@ -223,12 +220,12 @@ module Aspera
             target_file=call_data[:save_to_file]
             # override user's path to path in header
             if !response['Content-Disposition'].nil? and m=response['Content-Disposition'].match(/filename="([^"]+)"/)
-              target_file=m[1]
+              target_file=File.join(File.dirname(target_file),m[1])
             end
             # download with temp filename
-            target_file_tmp=target_file+'.http.partial'
+            target_file_tmp="#{target_file}#{@@download_partial_suffix}"
             Log.log.debug("saving to: #{target_file}")
-            File.open(target_file_tmp, "wb") do |file|
+            File.open(target_file_tmp, 'wb') do |file|
               result[:http].read_body do |fragment|
                 file.write(fragment)
                 new_process=progress.progress+fragment.length
@@ -239,7 +236,7 @@ module Aspera
             # rename at the end
             File.rename(target_file_tmp, target_file)
             progress=nil
-          end
+          end # save_to_file
         end
         # sometimes there is a ITF8 char (e.g. (c) )
         result[:http].body.force_encoding("UTF-8") if result[:http].body.is_a?(String)
@@ -267,10 +264,24 @@ module Aspera
           end
           Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
           retry unless (oauth_tries -= 1).zero?
-        end # if
+        end # if oauth
+        # moved ?
+        if e.response.is_a?(Net::HTTPRedirection)
+          if tries_remain_redirect > 0
+            tries_remain_redirect-=1
+            Log.log.info("URL is moved: #{e.response['location']}")
+            raise e
+            # TODO: rebuild request with new location
+            #retry
+          else
+            raise "too many redirect"
+          end
+        else
+          raise e
+        end
         # raise exception if could not retry and not return error in result
         raise e unless call_data[:return_error]
-      end
+      end # begin request
       Log.log.debug("result=#{result}")
       return result
 

data/lib/aspera/secrets.rb

@@ -0,0 +1,20 @@
+module Aspera
+  # Manage secrets in CLI using secure way (encryption, wallet, etc...)
+  class Secrets
+    attr_accessor :default_secret,:all_secrets
+    def initialize()
+      @default_secret=nil
+      @all_secrets={}
+    end
+
+    def get_secret(id=nil,mandatory=true)
+      secret=@default_secret || @all_secrets[id]
+      raise "please provide secret for #{id}" if secret.nil? and mandatory
+      return secret
+    end
+
+    def get_secrets
+      return @all_secrets
+    end
+  end
+end

data/lib/aspera/temp_file_manager.rb

@@ -6,6 +6,11 @@ module Aspera
   # create a temp file name for a given folder
   # files can be deleted on process exit by calling cleanup
   class TempFileManager
+    SEC_IN_DAY=86400
+    # assume no transfer last longer than this
+    # (garbage collect file list which were not deleted after transfer)
+    FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
+    private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC
     include Singleton
     def initialize
       @created_files=[]
@@ -33,5 +38,19 @@ module Aspera
       username = Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user' rescue 'unknown_user'
       return new_file_path_in_folder(Etc.systmpdir,base_name+'_'+username+'_')
     end
+
+    def cleanup_expired(temp_folder)
+      # garbage collect undeleted files
+      Dir.entries(temp_folder).each do |name|
+        file_path=File.join(temp_folder,name)
+        age_sec=(Time.now - File.stat(file_path).mtime).to_i
+        # check age of file, delete too old
+        if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
+          Log.log.debug("garbage collecting #{name}")
+          File.delete(file_path)
+        end
+      end
+
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aspera-cli
 version: !ruby/object:Gem::Version
-  version: 4.0.0.pre2
+  version: 4.2.0
 platform: ruby
 authors:
 - Laurent Martin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-07 00:00:00.000000000 Z
+date: 2021-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: xml-simple
@@ -58,28 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
-- !ruby/object:Gem::Dependency
-  name: mimemagic
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: execjs
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +122,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: websocket
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: websocket-client-simple
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -184,12 +198,14 @@ email:
 executables:
 - ascli
 - asession
+- dascli
 extensions: []
 extra_rdoc_files: []
 files:
 - README.md
 - bin/ascli
 - bin/asession
+- bin/dascli
 - docs/Makefile
 - docs/README.erb.md
 - docs/README.md
@@ -277,6 +293,7 @@ files:
 - lib/aspera/rest_call_error.rb
 - lib/aspera/rest_error_analyzer.rb
 - lib/aspera/rest_errors_aspera.rb
+- lib/aspera/secrets.rb
 - lib/aspera/ssh.rb
 - lib/aspera/sync.rb
 - lib/aspera/temp_file_manager.rb
@@ -290,6 +307,7 @@ metadata:
   source_code_uri: https://github.com/IBM/aspera-cli
   changelog_uri: https://github.com/IBM/aspera-cli
   rubygems_uri: https://rubygems.org/gems/aspera-cli
+  documentation_uri: https://www.rubydoc.info/gems/aspera-cli
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -301,12 +319,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements:
-- IBM Aspera ascp installed for the user
-rubygems_version: 3.0.3
+- Read the manual for any requirement
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: 'Execute actions using command line on IBM Aspera Server products: Aspera