aspera-cli 4.3.0 → 4.4.0

data/docs/doc_tools.rb

@@ -0,0 +1,58 @@
+# get transfer spec parameter description
+$LOAD_PATH.unshift(ENV["INCL_DIR_GEM"])
+require 'aspera/fasp/parameters'
+
+# check that required env vars exist, and files
+%w{EXENAME GEMSPEC INCL_USAGE INCL_COMMANDS INCL_ASESSION INCL_DIR_GEM}.each do |e|
+  raise "missing env var #{e}" unless ENV.has_key?(e)
+  raise "missing file #{ENV[e]}" unless File.exist?(ENV[e]) or !e.start_with?('INCL_')
+end
+
+# set values used in ERB
+# just command name
+def cmd;ENV['EXENAME'];end
+
+# used in text with formatting of command
+def tool;'`'+cmd+'`';end
+
+# prefix for env vars
+def evp;cmd.upcase+'_';end
+
+# just the name for "option preset"
+def opprst;'option preset';end
+
+# name with link
+def prst;'['+opprst+'](#lprt)';end
+
+# name with link (plural)
+def prsts;'['+opprst+'s](#lprt)';end
+
+# in title
+def prstt;opprst.capitalize;end
+
+def gemspec;Gem::Specification::load(ENV['GEMSPEC']) or raise "error loading #{ENV["GEMSPEC"]}";end
+
+def geminstadd;gemspec.version.to_s.match(/\.[^0-9]/)?' --pre':'';end
+
+# transfer spec description generation
+def spec_table
+  r='<table><tr><th>Field</th><th>Type</th>'
+  Aspera::Fasp::Parameters::SUPPORTED_AGENTS_SHORT.each do |c|
+    r << '<th>'<<c.to_s.upcase<<'</th>'
+  end
+  r << '<th>Description</th></tr>'
+  Aspera::Fasp::Parameters.man_table.each do |p|
+    p[:description] << (p[:description].empty? ? '' : "\n") << "(" << p[:cli] << ")" unless p[:cli].to_s.empty?
+    p.delete(:cli)
+    p.keys.each{|c|p[c]='&nbsp;' if p[c].to_s.empty?}
+    p[:description].gsub!("\n",'<br/>')
+    p[:type].gsub!(',','<br/>')
+    r << '<tr><td>'<<p[:name]<<'</td><td>'<<p[:type]<<'</td>'
+    Aspera::Fasp::Parameters::SUPPORTED_AGENTS_SHORT.each do |c|
+      r << '<td>'<<p[c]<<'</td>'
+    end
+    r << '<td>'<<p[:description]<<'</td></tr>'
+  end
+  r << '</table>'
+  return r
+end

data/lib/aspera/aoc.rb

@@ -2,6 +2,7 @@ require 'aspera/log'
 require 'aspera/rest'
 require 'aspera/hash_ext'
 require 'aspera/data_repository'
+require 'aspera/node'
 require 'base64'
 
 module Aspera
@@ -24,9 +25,9 @@ module Aspera
     JWT_AUDIENCE='https://api.asperafiles.com/api/v1/oauth2/token'
     OAUTH_API_SUBPATH='api/v1/oauth2'
     DEFAULT_TSPEC_INFO={
-      'remote_user' => 'xfer',
-      'ssh_port'    => 33001,
-      'fasp_port'   => 33001
+      'remote_user' => Node::ACCESS_KEY_TRANSFER_USER,
+      'ssh_port'    => Node::SSH_PORT_DEFAULT,
+      'fasp_port'   => Node::UDP_PORT_DEFAULT
     }
 
     private_constant :PRODUCT_NAME,:PROD_DOMAIN,:MAX_REDIRECT,:CLIENT_APPS,:PATHS_PUBLIC_LINK,:JWT_AUDIENCE,:OAUTH_API_SUBPATH,:DEFAULT_TSPEC_INFO

data/lib/aspera/cli/plugin.rb

@@ -2,10 +2,15 @@ module Aspera
   module Cli
     # base class for plugins modules
     class Plugin
+      # operation without id
       GLOBAL_OPS=[:create,:list]
+      # operation on specific instance
       INSTANCE_OPS=[:modify,:delete,:show]
       ALL_OPS=[GLOBAL_OPS,INSTANCE_OPS].flatten
-      #private_constant :GLOBAL_OPS,:INSTANCE_OPS,:ALL_OPS
+      # max number of items for list command
+      MAX_ITEMS='max'
+      # max number of pages for list command
+      MAX_PAGES='pmax'
 
       @@done=false
 
@@ -28,7 +33,7 @@ module Aspera
         end
       end
 
-      def entity_command(command,rest_api,res_class_path,display_fields,id_symb,id_default=nil,subkey=false)
+      def entity_command(command,rest_api,res_class_path,display_fields,id_symb,id_default=nil,use_subkey=false)
         if INSTANCE_OPS.include?(command)
           begin
             one_res_id=self.options.get_option(id_symb,:mandatory)
@@ -54,10 +59,11 @@ module Aspera
         when :list
           resp=rest_api.read(res_class_path,parameters)
           data=resp[:data]
+          # TODO: not generic : which application is this for ?
           if resp[:http]['Content-Type'].start_with?('application/vnd.api+json')
-            data=resp[:data][res_class_path]
+            data=data[res_class_path]
           end
-          data=data[res_class_path] if subkey
+          data=data[res_class_path] if use_subkey
           return {:type => :object_list, :data=>data, :fields=>display_fields}
         when :modify
           property=self.options.get_option(:property,:optional)
@@ -74,12 +80,13 @@ module Aspera
       end
 
       # implement generic rest operations on given resource path
-      def entity_action(rest_api,res_class_path,display_fields,id_symb,id_default=nil,subkey=false)
+      def entity_action(rest_api,res_class_path,display_fields,id_symb,id_default=nil,use_subkey=false)
         #res_name=res_class_path.gsub(%r{^.*/},'').gsub(%r{s$},'').gsub('_',' ')
         command=self.options.get_next_command(ALL_OPS)
-        return entity_command(command,rest_api,res_class_path,display_fields,id_symb,id_default,subkey)
+        return entity_command(command,rest_api,res_class_path,display_fields,id_symb,id_default,use_subkey)
       end
 
+      # shortcuts for plugin environment
       def options; return @agents[:options];end
 
       def transfer; return @agents[:transfer];end

data/lib/aspera/cli/plugins/aoc.rb

@@ -205,7 +205,7 @@ module Aspera
             node_api.call({:operation=>'GET',:subpath=>"files/#{node_file[:file_id]}/content",:save_to_file=>File.join(self.transfer.destination_folder('receive'),file_name)})
             return Main.result_status("downloaded: #{file_name}")
           when :v3
-            # Note: other "common" actions are unauthorized with user scope
+            # Note: other common actions are unauthorized with user scope
             command_legacy=self.options.get_next_command(Node::SIMPLE_ACTIONS)
             # TODO: shall we support all methods here ? what if there is a link ?
             node_api=@api_aoc.get_node_api(top_node_file[:node_info],scope: AoC::SCOPE_NODE_USER)
@@ -224,14 +224,14 @@ module Aspera
               items=node_api.read("files/#{node_file[:file_id]}")[:data]
               return {:type=>:single_object,:data=>items}
             when :modify
-              update_param=self.options.get_next_argument("update data (Hash)")
+              update_param=self.options.get_next_argument('update data (Hash)')
               res=node_api.update("files/#{node_file[:file_id]}",update_param)[:data]
               return {:type=>:single_object,:data=>res}
             when :permission
               command_perm=self.options.get_next_command([:list,:create])
               case command_perm
               when :list
-                # generic options : TODO: as arg ?
+                # generic options : TODO: as arg ? option_url_query
                 list_options||={'include'=>['[]','access_level','permission_count']}
                 # special value: ALL will show all permissions
                 if !VAL_ALL.eql?(node_file[:file_id])
@@ -239,11 +239,10 @@ module Aspera
                   list_options['file_id']=node_file[:file_id]
                   list_options['inherited']||=false
                 end
-                #option_url_query
                 items=node_api.read('permissions',list_options)[:data]
                 return {:type=>:object_list,:data=>items}
               when :create
-                #create_param=self.options.get_next_argument("creation data (Hash)")
+                #create_param=self.options.get_next_argument('creation data (Hash)')
                 set_workspace_info
                 access_id="ASPERA_ACCESS_KEY_ADMIN_WS_#{@workspace_id}"
                 node_file[:node_info]
@@ -266,14 +265,14 @@ module Aspera
                 return {:type=>:single_object,:data=>item}
               else raise "internal error:shall not reach here (#{command_perm})"
               end
-              raise "internal error:shall not reach here"
+              raise 'internal error:shall not reach here'
             else raise "internal error:shall not reach here (#{command_node_file})"
             end
-            raise "internal error:shall not reach here"
+            raise 'internal error:shall not reach here'
           when :permissions
 
           end # command_repo
-          raise "ERR"
+          raise 'ERR'
         end # execute_node_gen4_command
 
         # build constructor option list for AoC based on options of CLI
@@ -304,15 +303,15 @@ module Aspera
 
           ws_name=self.options.get_option(:workspace,:optional)
           if ws_name.nil?
-            Log.log.debug("using default workspace".green)
+            Log.log.debug('using default workspace'.green)
             if @default_workspace_id.eql?(nil)
-              raise CliError,"no default workspace defined for user, please specify workspace"
+              raise CliError,'no default workspace defined for user, please specify workspace'
             end
             # get default workspace
             @workspace_id=@default_workspace_id
           else
             # lookup another workspace
-            wss=@api_aoc.read("workspaces",{'q'=>ws_name})[:data]
+            wss=@api_aoc.read('workspaces',{'q'=>ws_name})[:data]
             wss=wss.select { |i| i['name'].eql?(ws_name) }
             case wss.length
             when 0
@@ -320,14 +319,14 @@ module Aspera
             when 1
               @workspace_id=wss.first['id']
             else
-              raise "unexpected case"
+              raise 'unexpected case'
             end
           end
           @workspace_data=@api_aoc.read("workspaces/#{@workspace_id}")[:data]
           Log.log.debug("workspace_id=#{@workspace_id},@workspace_data=#{@workspace_data}".red)
 
           @workspace_name||=@workspace_data['name']
-          Log.log.info("current workspace is "+@workspace_name.red)
+          Log.log.info('current workspace is '+@workspace_name.red)
 
           # display workspace
           self.format.display_status("Current Workspace: #{@workspace_name.red}#{@workspace_id == @default_workspace_id ? ' (default)' : ''}")
@@ -343,7 +342,7 @@ module Aspera
           end
           home_node_id||=@workspace_data['home_node_id']||@workspace_data['node_id']
           home_file_id||=@workspace_data['home_file_id']
-          raise "node_id must be defined" if home_node_id.to_s.empty?
+          raise 'node_id must be defined' if home_node_id.to_s.empty?
           @home_node_file={
             node_info: @api_aoc.read("nodes/#{home_node_id}")[:data],
             file_id: home_file_id
@@ -355,7 +354,7 @@ module Aspera
 
         def do_bulk_operation(ids_or_one,success_msg,id_result='id',&do_action)
           ids_or_one=[ids_or_one] unless self.options.get_option(:bulk)
-          raise "expecting Array" unless ids_or_one.is_a?(Array)
+          raise 'expecting Array' unless ids_or_one.is_a?(Array)
           result_list=[]
           ids_or_one.each do |id|
             one={id_result=>id}
@@ -411,7 +410,8 @@ module Aspera
 
         # private
         def option_url_query(default)
-          query=self.options.get_option(:query,:optional)||default
+          query=self.options.get_option(:query,:optional)
+          query=default if query.nil?
           Log.log.debug("Query=#{query}".bg_red)
           begin
             # check it is suitable
@@ -428,6 +428,37 @@ module Aspera
           end
         end
 
+        # Call @api_aoc.read with same parameters, but use paging if necessary to get all results
+        def read_with_paging(resource_class_path,base_query)
+          raise "Query must be Hash" unless base_query.is_a?(Hash)
+          # set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
+          base_query['per_page']=1000 unless base_query.has_key?('per_page')
+          max_items=base_query[MAX_ITEMS]
+          base_query.delete(MAX_ITEMS)
+          max_pages=base_query[MAX_PAGES]
+          base_query.delete(MAX_PAGES)
+          item_list=[]
+          total_count=nil
+          current_page=base_query['page']
+          current_page=1 if current_page.nil?
+          page_count=0
+          loop do
+            query=base_query.clone
+            query['page']=current_page
+            result=@api_aoc.read(resource_class_path,query)
+            total_count=result[:http]['X-Total-Count']
+            page_count+=1
+            current_page+=1
+            add_items=result[:data]
+            break if add_items.empty?
+            # append new items to full list
+            item_list += add_items
+            break if !max_pages.nil? and page_count > max_pages
+            break if !max_items.nil? and item_list.count > max_items
+          end
+          return item_list,total_count
+        end
+
         def execute_admin_action
           @api_aoc.oauth.params[:scope]=AoC::SCOPE_FILES_ADMIN
           command_admin=self.options.get_next_command([ :ats, :resource, :usage_reports, :analytics, :subscription, :auth_providers ])
@@ -517,11 +548,11 @@ module Aspera
               when 'organizations'; c_user_info['organization_id']
               when 'users'; c_user_info['id']
               when 'nodes'; c_user_info['id']
-              else raise "organizations or users for option --name"
+              else raise 'organizations or users for option --name'
               end
               #
               filter=self.options.get_option(:query,:optional) || {}
-              raise "query must be Hash" unless filter.is_a?(Hash)
+              raise 'query must be Hash' unless filter.is_a?(Hash)
               filter['limit']||=100
               if self.options.get_option(:once_only,:mandatory)
                 saved_date=[]
@@ -547,13 +578,13 @@ module Aspera
               return {:type=>:object_list,:data=>events}
             end
           when :resource
-            resource_type=self.options.get_next_argument('resource',[:self,:organization,:user,:group,:client,:contact,:dropbox,:node,:operation,:package,:saml_configuration, :workspace, :dropbox_membership,:short_link,:workspace_membership,:apps_new,:client_registration_token,:client_access_key,:kms_profile])
+            resource_type=self.options.get_next_argument('resource',[:self,:organization,:user,:group,:client,:contact,:dropbox,:node,:operation,:package,:saml_configuration, :workspace, :dropbox_membership,:short_link,:workspace_membership,:application,:client_registration_token,:client_access_key,:kms_profile])
             # get path on API
             resource_class_path=case resource_type
             when :self,:organization
-              "#{resource_type}"
-            when :apps_new
-              "admin/#{resource_type}"
+              resource_type
+            when :application
+              'admin/apps_new'
             when :dropbox
               resource_type.to_s+'es'
             when :client_registration_token,:client_access_key
@@ -582,13 +613,13 @@ module Aspera
                 res_id=@home_node_file[:node_info]['id']
               end
               if !res_name.nil?
-                Log.log.warn("name overrides id") unless res_id.nil?
+                Log.log.warn('name overrides id') unless res_id.nil?
                 matching=@api_aoc.read(resource_class_path,{:q=>res_name})[:data]
-                raise CliError,"no resource match name" if matching.empty?
+                raise CliError,'no resource match name' if matching.empty?
                 raise CliError,"several resources match name (#{matching.join(',')})" unless matching.length.eql?(1)
                 res_id=matching.first['id']
               end
-              raise CliBadArgument,"provide either id or name" if res_id.nil?
+              raise CliBadArgument,'provide either id or name' if res_id.nil?
               resource_instance_path="#{resource_class_path}/#{res_id}"
             end
             resource_instance_path=resource_class_path if singleton_object
@@ -598,26 +629,29 @@ module Aspera
               id_result='token' if resource_class_path.eql?('admin/client_registration_tokens')
               # TODO: report inconsistency: creation url is !=, and does not return id.
               resource_class_path='admin/client_registration/token' if resource_class_path.eql?('admin/client_registration_tokens')
-              list_or_one=self.options.get_next_argument("creation data (Hash)")
+              list_or_one=self.options.get_next_argument('creation data (Hash)')
               return do_bulk_operation(list_or_one,'created',id_result)do|params|
-                raise "expecting Hash" unless params.is_a?(Hash)
+                raise 'expecting Hash' unless params.is_a?(Hash)
                 @api_aoc.create(resource_class_path,params)[:data]
               end
             when :list
-              default_fields=['id','name']
-              list_query=nil
+              default_fields=['id']
+              default_query={}
               case resource_type
-              when :node; default_fields.push('host','access_key')
+              when :application; default_query={:organization_apps=>true};default_fields.push('app_type','app_name','available','direct_authorizations_allowed','workspace_authorizations_allowed')
+              when :client,:client_access_key,:dropbox,:group,:package,:saml_configuration,:workspace; default_fields.push('name')
+              when :client_registration_token; default_fields.push('value','data.client_subject_scopes','created_at')
+              when :contact; default_fields=['email','name','source_id','source_type']
+              when :node; default_fields.push('name','host','access_key')
               when :operation; default_fields=nil
-              when :contact; default_fields=["email","name","source_id","source_type"]
-              when :apps_new; list_query={:organization_apps=>true};default_fields=['app_type','available']
-              when :client_registration_token; default_fields=['id','value','data.client_subject_scopes','created_at']
+              when :short_link; default_fields.push('short_url','data.url_token_data.purpose')
+              when :user; default_fields.push('name','email')
               end
-              result=@api_aoc.read(resource_class_path,option_url_query(list_query))
-              count_msg="Items: #{result[:data].length}/#{result[:http]['X-Total-Count']}"
-              count_msg=count_msg.bg_red unless result[:data].length.eql?(result[:http]['X-Total-Count'].to_i)
+              item_list,total_count=read_with_paging(resource_class_path,option_url_query(default_query))
+              count_msg="Items: #{item_list.length}/#{total_count}"
+              count_msg=count_msg.bg_red unless item_list.length.eql?(total_count.to_i)
               self.format.display_status(count_msg)
-              return {:type=>:object_list,:data=>result[:data],:fields=>default_fields}
+              return {:type=>:object_list,:data=>item_list,:fields=>default_fields}
             when :show
               object=@api_aoc.read(resource_instance_path)[:data]
               fields=object.keys.select{|k|!k.eql?('certificate')}
@@ -647,8 +681,8 @@ module Aspera
             when :shared_folders
               read_params = case resource_type
               when :workspace;{'access_id'=>"ASPERA_ACCESS_KEY_ADMIN_WS_#{res_id}",'access_type'=>'user'}
-              when :node;{'include'=>['[]','access_level','permission_count'],'created_by_id'=>"ASPERA_ACCESS_KEY_ADMIN"}
-              else raise "error"
+              when :node;{'include'=>['[]','access_level','permission_count'],'created_by_id'=>'ASPERA_ACCESS_KEY_ADMIN'}
+              else raise 'error'
               end
               res_data=@api_aoc.read("#{resource_class_path}/#{res_id}/permissions",read_params)[:data]
               fields=case resource_type
@@ -688,7 +722,7 @@ module Aspera
               create_data.deep_merge!(user_create_data)
               Log.dump(:data,create_data)
               raise :ERROR
-            else raise "unknown command"
+            else raise 'unknown command'
             end
           when :usage_reports
             return {:type=>:object_list,:data=>@api_aoc.read('usage_reports',{:workspace_id=>@workspace_id})[:data]}
@@ -719,7 +753,7 @@ module Aspera
             when :workspaces
               return {:type=>:object_list,:data=>@api_aoc.read('workspaces')[:data],:fields=>['id','name']}
               #              when :settings
-              #                return {:type=>:object_list,:data=>@api_aoc.read("client_settings/")[:data]}
+              #                return {:type=>:object_list,:data=>@api_aoc.read('client_settings/')[:data]}
             when :shared_inboxes
               query=option_url_query(nil)
               if query.nil?
@@ -764,7 +798,7 @@ module Aspera
               resolve_package_recipients(package_creation,'recipients')
               resolve_package_recipients(package_creation,'bcc_recipients')
 
-              #  create a new package with one file
+              #  create a new package container
               package_info=@api_aoc.create('packages',package_creation)[:data]
 
               #  get node information for the node on which package must be created
@@ -775,7 +809,7 @@ module Aspera
 
               # execute transfer
               node_file = {node_info: node_info, file_id: package_info['contents_file_id']}
-              # raise esception if at least one error
+              # raise exception if at least one error
               Main.result_transfer(transfer_start(AoC::PACKAGES_APP,'send',node_file,AoC.package_tags(package_info,'upload')))
               # return all info on package
               return { :type=>:single_object, :data =>package_info}
@@ -826,8 +860,10 @@ module Aspera
               package_info=@api_aoc.read("packages/#{package_id}")[:data]
               return { :type=>:single_object, :data =>package_info }
             when :list
-              # list all packages ('page'=>1,'per_page'=>10,)'sort'=>'-sent_at',
-              packages=@api_aoc.read('packages',{'archived'=>false,'exclude_dropbox_packages'=>true,'has_content'=>true,'received'=>true,'workspace_id'=>@workspace_id})[:data]
+              query=option_url_query({'archived'=>false,'exclude_dropbox_packages'=>true,'has_content'=>true,'received'=>true})
+              raise 'option must be Hash' unless query.is_a?(Hash)
+              query['workspace_id']||=@workspace_id
+              packages=@api_aoc.read('packages',query)[:data]
               return {:type=>:object_list,:data=>packages,:fields=>['id','name','bytes_transferred']}
             when :delete
               list_or_one=self.options.get_option(:id,:mandatory)
@@ -910,9 +946,9 @@ module Aspera
               end
               return result
             end # files command
-            throw "Error: shall not reach this line"
+            throw 'Error: shall not reach this line'
           when :automation
-            Log.log.warn("BETA: work under progress")
+            Log.log.warn('BETA: work under progress')
             # automation api is not in the same place
             automation_rest_params=@api_aoc.params.clone
             automation_rest_params[:base_url].gsub!('/api/','/automation/')
@@ -935,9 +971,9 @@ module Aspera
                 wf_command=self.options.get_next_command([:list,:create,:show])
                 wf_id=self.options.get_option(:id,:mandatory)
                 step=automation_api.create('steps',{'workflow_id'=>wf_id})[:data]
-                automation_api.update("workflows/#{wf_id}",{'step_order'=>[step["id"]]})
-                action=automation_api.create('actions',{'step_id'=>step["id"],'type'=>'manual'})[:data]
-                automation_api.update("steps/#{step["id"]}",{'action_order'=>[action["id"]]})
+                automation_api.update("workflows/#{wf_id}",{'step_order'=>[step['id']]})
+                action=automation_api.create('actions',{'step_id'=>step['id'],'type'=>'manual'})[:data]
+                automation_api.update("steps/#{step['id']}",{'action_order'=>[action['id']]})
                 wf=automation_api.read("workflows/#{wf_id}")[:data]
                 return {:type=>:single_object,:data=>wf}
               end
@@ -953,7 +989,7 @@ module Aspera
           else
             raise "internal error: #{command}"
           end # action
-          raise RuntimeError, "internal error: command shall return"
+          raise RuntimeError, 'internal error: command shall return'
         end
 
         private :c_user_info,:aoc_params,:set_workspace_info,:set_home_node_file,:do_bulk_operation,:resolve_package_recipients,:option_url_query,:assert_public_link_types,:execute_admin_action