aspera-cli 4.1.0 → 4.3.0

data/lib/aspera/cli/plugins/aoc.rb

@@ -5,6 +5,7 @@ require 'aspera/cli/transfer_agent'
 require 'aspera/aoc'
 require 'aspera/node'
 require 'aspera/persistency_action_once'
+require 'aspera/id_generator'
 require 'securerandom'
 require 'resolv'
 require 'date'
@@ -15,7 +16,6 @@ module Aspera
       class Aoc < BasicAuthPlugin
         # special value for package id
         VAL_ALL='ALL'
-        attr_reader :api_aoc
         def initialize(env)
           super(env)
           @default_workspace_id=nil
@@ -26,20 +26,20 @@ module Aspera
           @api_aoc=nil
           @url_token_data=nil
           @user_info=nil
-          self.options.add_opt_list(:auth,Oauth.auth_types,'type of Oauth authentication')
+          @api_aoc=nil
+          self.options.add_opt_list(:auth,Oauth.auth_types,'OAuth type of authentication')
           self.options.add_opt_list(:operation,[:push,:pull],'client operation for transfers')
-          self.options.add_opt_simple(:client_id,'API client identifier in application')
-          self.options.add_opt_simple(:client_secret,'API client passcode')
-          self.options.add_opt_simple(:redirect_uri,'API client redirect URI')
-          self.options.add_opt_simple(:private_key,'RSA private key PEM value for JWT (prefix file path with @val:@file:)')
+          self.options.add_opt_simple(:client_id,'OAuth API client identifier in application')
+          self.options.add_opt_simple(:client_secret,'OAuth API client passcode')
+          self.options.add_opt_simple(:redirect_uri,'OAuth API client redirect URI')
+          self.options.add_opt_simple(:private_key,'OAuth JWT RSA private key PEM value (prefix file path with @val:@file:)')
           self.options.add_opt_simple(:workspace,'name of workspace')
           self.options.add_opt_simple(:name,'resource name')
           self.options.add_opt_simple(:path,'file or folder path')
           self.options.add_opt_simple(:link,'public link to shared resource')
           self.options.add_opt_simple(:new_user_option,'new user creation option')
           self.options.add_opt_simple(:from_folder,'share to share source folder')
-          self.options.add_opt_simple(:scope,'scope for AoC API calls')
-          self.options.add_opt_simple(:notify,'notify users that file was received')
+          self.options.add_opt_simple(:scope,'OAuth scope for AoC API calls')
           self.options.add_opt_boolean(:bulk,'bulk operation')
           self.options.add_opt_boolean(:default_ports,'use standard FASP ports or get from node api')
           self.options.set_option(:bulk,:no)
@@ -52,19 +52,15 @@ module Aspera
           self.options.parse_options!
           AoC.set_use_default_ports(self.options.get_option(:default_ports))
           return if env[:man_only]
-          @api_aoc=AoC.new(aoc_params('api/v1'))
-          # add access key secrets
-          @api_aoc.add_secrets(self.config.get_secrets)
         end
 
-        # call this to populate single AK secret in AoC API object, from options
-        # make sure secret is available
-        def find_ak_secret(ak,mandatory=true)
-          # secret hash is already provisioned
-          # optionally override with specific secret
-          @api_aoc.add_secrets({ak=>self.config.get_secret(ak,mandatory)})
-          # check that secret was provided as single value or dictionary
-          raise CliBadArgument,"Please provide option secret or entry in option secrets for: #{ak}" unless @api_aoc.has_secret(ak) or !mandatory
+        def get_api
+          if @api_aoc.nil?
+            @api_aoc=AoC.new(aoc_params(AoC::API_V1))
+            # add keychain for access key secrets
+            @api_aoc.key_chain=@agents[:secret]
+          end
+          return @api_aoc
         end
 
         # cached user information
@@ -87,19 +83,29 @@ module Aspera
           return self.transfer.start(*@api_aoc.tr_spec(app,direction,node_file,ts_add))
         end
 
-        NODE4_COMMANDS=[ :browse, :find, :mkdir, :rename, :delete, :upload, :download, :transfer, :http_node_download, :v3, :file, :bearer_token_node ]
+        NODE4_COMMANDS=[ :browse, :find, :mkdir, :rename, :delete, :upload, :download, :transfer, :http_node_download, :v3, :file, :bearer_token_node, :node_info ]
 
         def execute_node_gen4_command(command_repo,top_node_file)
           case command_repo
           when :bearer_token_node
             thepath=self.options.get_next_argument('path')
             node_file = @api_aoc.resolve_node_file(top_node_file,thepath)
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER, use_secret: false)
             return Main.result_status(node_api.oauth_token)
+          when :node_info
+            thepath=self.options.get_next_argument('path')
+            node_file = @api_aoc.resolve_node_file(top_node_file,thepath)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER, use_secret: false)
+            return {:type=>:single_object,:data=>{
+              url: node_file[:node_info]['url'],
+              username: node_file[:node_info]['access_key'],
+              password: node_api.oauth_token,
+              root_id: node_file[:file_id]
+              }}
           when :browse
             thepath=self.options.get_next_argument('path')
             node_file = @api_aoc.resolve_node_file(top_node_file,thepath)
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             file_info = node_api.read("files/#{node_file[:file_id]}")[:data]
             if file_info['type'].eql?('folder')
               result=node_api.read("files/#{node_file[:file_id]}/files",self.options.get_option(:value,:optional))
@@ -111,28 +117,22 @@ module Aspera
             return {:type=>:object_list,:data=>items,:fields=>['name','type','recursive_size','size','modified_time','access_level']}
           when :find
             thepath=self.options.get_next_argument('path')
-            exec_prefix='exec:'
-            expression=self.options.get_option(:value,:optional)||"#{exec_prefix}true"
             node_file=@api_aoc.resolve_node_file(top_node_file,thepath)
-            if expression.start_with?(exec_prefix)
-              test_block=eval "lambda{|f|#{expression[exec_prefix.length..-1]}}"
-            else
-              test_block=lambda{|f|f['name'].match(/#{expression}/)}
-            end
+            test_block=Aspera::Node.file_matcher(self.options.get_option(:value,:optional))
             return {:type=>:object_list,:data=>@api_aoc.find_files(node_file,test_block),:fields=>['path']}
           when :mkdir
             thepath=self.options.get_next_argument('path')
             containing_folder_path = thepath.split(AoC::PATH_SEPARATOR)
             new_folder=containing_folder_path.pop
             node_file = @api_aoc.resolve_node_file(top_node_file,containing_folder_path.join(AoC::PATH_SEPARATOR))
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             result=node_api.create("files/#{node_file[:file_id]}/files",{:name=>new_folder,:type=>:folder})[:data]
             return Main.result_status("created: #{result['name']} (id=#{result['id']})")
           when :rename
             thepath=self.options.get_next_argument('source path')
             newname=self.options.get_next_argument('new name')
             node_file = @api_aoc.resolve_node_file(top_node_file,thepath)
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             result=node_api.update("files/#{node_file[:file_id]}",{:name=>newname})[:data]
             return Main.result_status("renamed #{thepath} to #{newname}")
           when :delete
@@ -140,7 +140,7 @@ module Aspera
             return do_bulk_operation(thepath,'deleted','path') do |l_path|
               raise "expecting String (path), got #{l_path.class.name} (#{l_path})" unless l_path.is_a?(String)
               node_file = @api_aoc.resolve_node_file(top_node_file,l_path)
-              node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+              node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
               result=node_api.delete("files/#{node_file[:file_id]}")[:data]
               {'path'=>l_path}
             end
@@ -163,7 +163,7 @@ module Aspera
             client_node_file = @api_aoc.resolve_node_file(client_home_node_file,client_folder)
             server_node_file = @api_aoc.resolve_node_file(server_home_node_file,server_folder)
             # force node as transfer agent
-            @agents[:transfer].set_agent_instance(Fasp::Node.new(@api_aoc.get_node_api(client_node_file[:node_info],AoC::SCOPE_NODE_USER)))
+            @agents[:transfer].set_agent_instance(Fasp::Node.new(@api_aoc.get_node_api(client_node_file[:node_info],scope: AoC::SCOPE_NODE_USER)))
             # additional node to node TS info
             add_ts={
               'remote_access_key'   => server_node_file[:node_info]['access_key'],
@@ -201,14 +201,14 @@ module Aspera
             raise CliBadArgument,'one file at a time only in HTTP mode' if source_paths.length > 1
             file_name = source_paths.first['source']
             node_file = @api_aoc.resolve_node_file(top_node_file,File.join(source_folder,file_name))
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             node_api.call({:operation=>'GET',:subpath=>"files/#{node_file[:file_id]}/content",:save_to_file=>File.join(self.transfer.destination_folder('receive'),file_name)})
             return Main.result_status("downloaded: #{file_name}")
           when :v3
             # Note: other "common" actions are unauthorized with user scope
             command_legacy=self.options.get_next_command(Node::SIMPLE_ACTIONS)
             # TODO: shall we support all methods here ? what if there is a link ?
-            node_api=@api_aoc.get_node_api(top_node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(top_node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: node_api)).execute_action(command_legacy)
           when :file
             file_path=self.options.get_option(:path,:optional)
@@ -217,7 +217,7 @@ module Aspera
             else
               {node_info: top_node_file[:node_info],file_id: self.options.get_option(:id,:mandatory)}
             end
-            node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+            node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
             command_node_file=self.options.get_next_command([:show,:permission,:modify])
             case command_node_file
             when :show
@@ -528,7 +528,7 @@ module Aspera
                 startdate_persistency=PersistencyActionOnce.new(
                 manager: @agents[:persistency],
                 data: saved_date,
-                ids:  ['aoc_ana_date',self.options.get_option(:url,:mandatory),@workspace_name].push(filter_resource,filter_id))
+                ids: IdGenerator.from_list(['aoc_ana_date',self.options.get_option(:url,:mandatory),@workspace_name].push(filter_resource,filter_id)))
                 start_datetime=saved_date.first
                 stop_datetime=Time.now.utc.strftime('%FT%T.%LZ')
                 #Log.log().error("start: #{start_datetime}")
@@ -537,18 +537,11 @@ module Aspera
                 filter['start_time'] = start_datetime unless start_datetime.nil?
                 filter['stop_time'] = stop_datetime
               end
-              notification=self.options.get_option(:notify,:optional)
               events=analytics_api.read("#{filter_resource}/#{filter_id}/#{event_type}",option_url_query(filter))[:data][event_type]
               startdate_persistency.save unless startdate_persistency.nil?
-              if !notification.nil?
-                require 'erb'
-                events.each do |transfer|
-                  email_to_send={}
-                  notification.each do |k,v|
-                    email_to_send[k.to_sym]=ERB.new(v).result(binding)
-                  end
-                  Log.log().error("send email:   #{email_to_send}")
-                  self.config.send_email(email_to_send)
+              if !self.options.get_option(:notif_to,:optional).nil?
+                events.each do |tr_event|
+                  self.config.send_email_template({ev: tr_event})
                 end
               end
               return {:type=>:object_list,:data=>events}
@@ -646,7 +639,6 @@ module Aspera
               return Main.result_success
             when :v3,:v4
               res_data=@api_aoc.read(resource_instance_path)[:data]
-              find_ak_secret(res_data['access_key'])
               api_node=@api_aoc.get_node_api(res_data)
               return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).execute_action if command.eql?(:v3)
               ak_data=api_node.call({:operation=>'GET',:subpath=>"access_keys/#{res_data['access_key']}",:headers=>{'Accept'=>'application/json'}})[:data]
@@ -672,7 +664,7 @@ module Aspera
               res_data=@api_aoc.read(resource_instance_path)[:data]
               node_file = @api_aoc.resolve_node_file({node_info: res_data, file_id: ak_data['root_file_id']},folder_path)
 
-              #node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+              #node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
               #file_info = node_api.read("files/#{node_file[:file_id]}")[:data]
 
               access_id="ASPERA_ACCESS_KEY_ADMIN_WS_#{@workspace_id}"
@@ -704,16 +696,17 @@ module Aspera
         end
 
         # must be public
-        ACTIONS=[ :apiinfo, :bearer_token, :organization, :tier_restrictions, :user, :workspace, :packages, :files, :gateway, :admin, :automation, :servers]
+        ACTIONS=[ :reminder, :bearer_token, :organization, :tier_restrictions, :user, :workspace, :packages, :files, :gateway, :admin, :automation, :servers]
 
         def execute_action
           command=self.options.get_next_command(ACTIONS)
+          get_api unless [:reminder,:servers].include?(command)
           case command
-          when :apiinfo
-            api_info={}
-            num=1
-            Resolv::DNS.open{|dns|dns.each_address('api.ibmaspera.com'){|a| api_info["api.#{num}"]=a;num+=1}}
-            return {:type=>:single_object,:data=>api_info}
+          when :reminder
+            # send an email reminder with list of orgs
+            user_email=options.get_option(:username,:mandatory)
+            Rest.new(base_url: "#{AoC.api_base_url}/#{AoC::API_V1}").create('organization_reminders',{email: user_email})[:data]
+            return Main.result_status("List of organizations user is member of, has been sent by e-mail to #{user_email}")
           when :bearer_token
             return {:type=>:text,:data=>@api_aoc.oauth_token}
           when :organization
@@ -799,7 +792,7 @@ module Aspera
                 skip_ids_persistency=PersistencyActionOnce.new(
                 manager: @agents[:persistency],
                 data: skip_ids_data,
-                ids:  ['aoc_recv',self.options.get_option(:url,:mandatory),@workspace_id].push(*@persist_ids))
+                id: IdGenerator.from_list(['aoc_recv',self.options.get_option(:url,:mandatory),@workspace_id].push(*@persist_ids)))
               end
               if ids_to_download.eql?(VAL_ALL)
                 # get list of packages in inbox
@@ -820,14 +813,14 @@ module Aspera
                 add_ts={'paths'=>[{'source'=>'.'}]}
                 node_file = {node_info: node_info, file_id: package_info['contents_file_id']}
                 statuses=transfer_start(AoC::PACKAGES_APP,'receive',node_file,AoC.package_tags(package_info,'download').merge(add_ts))
-                result_transfer.push({'package'=>package_id,'status'=>statuses.map{|i|i.to_s}.join(',')})
+                result_transfer.push({'package'=>package_id,Main::STATUS_FIELD=>statuses})
                 # update skip list only if all transfer sessions completed
                 if TransferAgent.session_status(statuses).eql?(:success)
                   skip_ids_data.push(package_id)
                   skip_ids_persistency.save unless skip_ids_persistency.nil?
                 end
               end
-              return {:type=>:object_list,:data=>result_transfer}
+              return Main.result_transfer_multiple(result_transfer)
             when :show
               package_id=self.options.get_next_argument('package ID')
               package_info=@api_aoc.read("packages/#{package_id}")[:data]
@@ -847,7 +840,6 @@ module Aspera
             # get workspace related information
             set_workspace_info
             set_home_node_file
-            find_ak_secret(@home_node_file[:node_info]['access_key'],false)
             command_repo=self.options.get_next_command(NODE4_COMMANDS.clone.concat([:short_link]))
             case command_repo
             when *NODE4_COMMANDS; return execute_node_gen4_command(command_repo,@home_node_file)
@@ -894,7 +886,7 @@ module Aspera
               end
               result=self.entity_action(@api_aoc,'short_links',nil,:id,'self')
               if result[:data].is_a?(Hash) and result[:data].has_key?('created_at') and result[:data]['resource_type'].eql?('UrlToken')
-                node_api=@api_aoc.get_node_api(node_file[:node_info],AoC::SCOPE_NODE_USER)
+                node_api=@api_aoc.get_node_api(node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
                 # TODO: access level as arg
                 access_levels=Aspera::Node::ACCESS_LEVELS #['delete','list','mkdir','preview','read','rename','write']
                 perm_data={
@@ -957,17 +949,14 @@ module Aspera
           when :admin
             return execute_admin_action
           when :servers
-            self.format.display_status("Beta feature: #{@api_aoc.params[:base_url]}")
-            server_api=Rest.new(base_url: @api_aoc.params[:base_url])
-            servers=server_api.read('servers')[:data]
-            return {:type=>:object_list,:data=>servers}
+            return {:type=>:object_list,:data=>Rest.new(base_url: "#{AoC.api_base_url}/#{AoC::API_V1}").read('servers')[:data]}
           else
             raise "internal error: #{command}"
           end # action
           raise RuntimeError, "internal error: command shall return"
         end
 
-        private :find_ak_secret,:c_user_info,:aoc_params,:set_workspace_info,:set_home_node_file,:do_bulk_operation,:resolve_package_recipients,:option_url_query,:assert_public_link_types,:execute_admin_action
+        private :c_user_info,:aoc_params,:set_workspace_info,:set_home_node_file,:do_bulk_operation,:resolve_package_recipients,:option_url_query,:assert_public_link_types,:execute_admin_action
         private_constant :VAL_ALL,:NODE4_COMMANDS
 
       end # AoC

data/lib/aspera/cli/plugins/ats.rb

@@ -107,7 +107,7 @@ module Aspera
               :auth     => {
               :type     => :basic,
               :username => access_key_id,
-              :password => self.config.get_secret(access_key_id)}})
+              :password => @agents[:secret].get_secret(access_key_id)}})
             command=self.options.get_next_command(Node::COMMON_ACTIONS)
             return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_node)).execute_action(command)
           when :cluster
@@ -116,7 +116,7 @@ module Aspera
               :auth     => {
               :type     => :basic,
               :username => access_key_id,
-              :password => self.config.get_secret(access_key_id)
+              :password => @agents[:secret].get_secret(access_key_id)
               }}
             api_ak_auth=Rest.new(rest_params)
             return {:type=>:single_object, :data=>api_ak_auth.read("servers")[:data]}