aspera-cli 4.0.0.pre1

data/lib/aspera/cli/plugins/server.rb

@@ -0,0 +1,216 @@
+require 'aspera/cli/basic_auth_plugin'
+require 'aspera/ascmd'
+require 'aspera/ssh'
+require 'aspera/nagios'
+require 'tempfile'
+
+module Aspera
+  module Cli
+    module Plugins
+      # implement basic remote access with FASP/SSH
+      class Server < BasicAuthPlugin
+        class LocalExecutor
+          def execute(cmd,input=nil)
+            `#{cmd}`
+          end
+        end
+
+        def initialize(env)
+          super(env)
+          self.options.add_opt_simple(:ssh_keys,'ssh key path list (Array or single)')
+          self.options.add_opt_simple(:ssh_options,'ssh options (Hash)')
+          self.options.add_opt_simple(:cmd_prefix,'prefix to add for as cmd execution, e.g. sudo or /opt/aspera/bin ')
+          self.options.set_option(:ssh_keys,[])
+          self.options.set_option(:ssh_options,{})
+          self.options.parse_options!
+        end
+
+        def key_symb_to_str_single(source)
+          return source.inject({}){|memo,(k,v)| memo[k.to_s] = v; memo}
+        end
+
+        def key_symb_to_str_list(source)
+          return source.map{|o| key_symb_to_str_single(o)}
+        end
+
+        def asctl_parse(text)
+          r=/:\s*/
+          return text.split("\n").map do |line|
+            # console
+            line.gsub!(/(SessionDataCollector)/,'\1 ')
+            # orchestrator
+            line.gsub!(/ with pid:.*/,'')
+            line.gsub!(/ is /,': ')
+            x=line.split(r)
+            next unless x.length.eql?(2)
+            y={'process'=>x.first,'state'=>x.last}
+            # console
+            y['state'].gsub!(/\.+$/,'')
+            # console
+            y['process'].gsub!(/^.+::/,'')
+            # faspex
+            y['process'].gsub!(/^Faspex /,'')
+            # faspex
+            y['process'].gsub!(/ Background/,'')
+            y['process'].gsub!(/serving orchestrator on port /,'')
+            # console
+            r=/\s+/ if y['process'].eql?('Console')
+            # orchestrator
+            y['process'].gsub!(/^  -> /,'')
+            y['process'].gsub!(/ Process/,'')
+            y
+          end.select{|i|!i.nil?}
+        end
+
+        ACTIONS=[:nagios,:nodeadmin,:userdata,:configurator,:ctl,:download,:upload,:browse,:delete,:rename].concat(Aspera::AsCmd::OPERATIONS)
+
+        def execute_action
+          server_uri=URI.parse(self.options.get_option(:url,:mandatory))
+          Log.log.debug("URI : #{server_uri}, port=#{server_uri.port}, scheme:#{server_uri.scheme}")
+          shell_executor=nil
+          case server_uri.scheme
+          when 'ssh'
+            server_transfer_spec={
+              'remote_host'=>server_uri.hostname,
+              'remote_user'=>self.options.get_option(:username,:mandatory),
+            }
+            ssh_options=self.options.get_option(:ssh_options,:optional)
+            raise 'expecting a Hash for ssh_options' unless ssh_options.is_a?(Hash)
+            if !server_uri.port.nil?
+              ssh_options[:port]=server_uri.port
+              server_transfer_spec['ssh_port']=server_uri.port
+            end
+            cred_set=false
+            password=self.options.get_option(:password,:optional)
+            if !password.nil?
+              ssh_options[:password]=password
+              server_transfer_spec['remote_password']=password
+              cred_set=true
+            end
+            ssh_keys=self.options.get_option(:ssh_keys,:optional)
+            if !ssh_keys.nil?
+              raise 'expecting single value or array for ssh_keys' unless ssh_keys.is_a?(Array) or ssh_keys.is_a?(String)
+              ssh_keys=[ssh_keys] if ssh_keys.is_a?(String)
+              ssh_keys.map!{|p|File.expand_path(p)}
+              Log.log.debug("ssh keys=#{ssh_keys}")
+              ssh_options[:keys]=ssh_keys
+              server_transfer_spec['EX_ssh_key_paths']=ssh_keys
+              ssh_keys.each do |k|
+                Log.log.warn("no such key file: #{k}") unless File.exist?(k)
+              end
+              cred_set=true
+            end
+            raise 'either password or key must be provided' if !cred_set
+            shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
+          when 'local'
+            shell_executor=LocalExecutor.new
+          else
+            raise CliError,'Only ssh scheme is supported in url' if !server_uri.scheme.eql?('ssh')
+          end
+
+          # get command and set aliases
+          command=self.options.get_next_command(ACTIONS)
+          command=:ls if command.eql?(:browse)
+          command=:rm if command.eql?(:delete)
+          command=:mv if command.eql?(:rename)
+          case command
+          when :nagios
+            nagios=Nagios.new
+            command_nagios=self.options.get_next_command([ :app_services, :transfer ])
+            case command_nagios
+            when :app_services
+              # will not work with aspshell, requires Linux/bash
+              procs=shell_executor.execute('ps -A -o comm').split("\n")
+              Log.log.debug("found: #{procs}")
+              ['asperanoded','asperaredisd'].each do |name|
+                nagios.add_critical('general',"missing process #{name}") unless procs.include?(name)
+              end
+              nagios.add_ok('daemons','ok') if nagios.data.empty?
+              return nagios.result
+            when :transfer
+              file = Tempfile.new('transfer_test')
+              filepath=file.path
+              file.write("This is a test file for transfer test")
+              file.close
+              probe_ts=server_transfer_spec.merge({
+                'direction'     => 'send',
+                'cookie'        => 'aspera.sync', # hide in console
+                'resume_policy' => 'none',
+                'paths'         => [{'source'=>filepath,'destination'=>'.fasping'}]
+              })
+              statuses=self.transfer.start(probe_ts,{:src=>:direct})
+              file.unlink
+              if TransferAgent.session_status(statuses).eql?(:success)
+                nagios.add_ok('transfer','ok')
+              else
+                nagios.add_critical('transfer',statuses.select{|i|!i.eql?(:success)}.first.to_s)
+              end
+            else raise "ERROR"
+            end
+            return nagios.result
+          when :nodeadmin,:userdata,:configurator,:ctl
+            realcmd='as'+command.to_s
+            prefix=self.options.get_option(:cmd_prefix,:optional)
+            if !prefix.nil?
+              realcmd="#{prefix}#{realcmd}"
+            end
+            args = self.options.get_next_argument("#{realcmd} arguments",:multiple)
+            result=shell_executor.execute(args.unshift(realcmd))
+            case command
+            when :ctl
+              return {:type=>:object_list,:data=>asctl_parse(result)}#
+            when :configurator
+              lines=result.split("\n")
+              Log.log.debug(`type asconfigurator`)
+              result=lines
+              if lines.first.eql?('success')
+                lines.shift
+                result={}
+                lines.each do |line|
+                  Log.log.debug("#{line}")
+                  data=line.split(',').map{|i|i.gsub(/^"/,'').gsub(/"$/,'')}.map{|i|case i;when'AS_NULL';nil;when'true';true;when'false';false;else i;end}
+                  Log.log.debug("#{data}")
+                  section=data.shift
+                  datapart=result[section]||={}
+                  if section.eql?('user')
+                    name=data.shift
+                    datapart=datapart[name]||={}
+                  end
+                  datapart=datapart[data.shift]={}
+                  datapart['default']=data.pop
+                  datapart['value']=data.pop
+                end
+                return {:type=>:single_object,:data=>result,:fields=>['section','name','value','default'],:option_expand_last=>true}
+              end
+            end
+            return Main.result_status(result)
+          when :upload
+            return Main.result_transfer(self.transfer.start(server_transfer_spec.merge('direction'=>'send'),{:src=>:direct}))
+          when :download
+            return Main.result_transfer(self.transfer.start(server_transfer_spec.merge('direction'=>'receive'),{:src=>:direct}))
+          when *Aspera::AsCmd::OPERATIONS
+            args=self.options.get_next_argument('ascmd command arguments',:multiple,:optional)
+            ascmd=Aspera::AsCmd.new(shell_executor)
+            begin
+              result=ascmd.send(:execute_single,command,args)
+              case command
+              when :mkdir;  return Main.result_success
+              when :mv;     return Main.result_success
+              when :cp;     return Main.result_success
+              when :rm;     return Main.result_success
+              when :ls;     return {:type=>:object_list,:data=>key_symb_to_str_list(result),:fields=>['zmode','zuid','zgid','size','mtime','name']}
+              when :info;   return {:type=>:single_object,:data=>key_symb_to_str_single(result)}
+              when :df;     return {:type=>:object_list,:data=>key_symb_to_str_list(result)}
+              when :du;     return {:type=>:single_object,:data=>key_symb_to_str_single(result)}
+              when :md5sum; return {:type=>:single_object,:data=>key_symb_to_str_single(result)}
+              end
+            rescue Aspera::AsCmd::Error => e
+              raise CliBadArgument,e.extended_message
+            end
+          else raise "internal error: unexpected action"
+          end
+        end # execute_action
+      end # Server
+    end # Plugins
+  end # Cli
+end # Aspera

data/lib/aspera/cli/plugins/shares.rb

@@ -0,0 +1,63 @@
+require 'aspera/cli/plugins/node'
+
+module Aspera
+  module Cli
+    module Plugins
+      class Shares < BasicAuthPlugin
+        def initialize(env)
+          super(env)
+          #self.options.parse_options!
+        end
+
+        ACTIONS=[ :repository,:admin ]
+
+        def execute_action
+          command=self.options.get_next_command(ACTIONS)
+          case command
+          when :repository
+            api_shares_node=basic_auth_api('node_api')
+            command=self.options.get_next_command(Node::COMMON_ACTIONS)
+            case command
+            when *Node::COMMON_ACTIONS; Node.new(@agents.merge(skip_basic_auth_options: true,node_api: api_shares_node)).execute_action(command)
+            else raise "INTERNAL ERROR, unknown command: [#{command}]"
+            end
+          when :admin
+            api_shares_admin=basic_auth_api('api/v1')
+            command=self.options.get_next_command([:user,:share])
+            case command
+            when :user
+              command=self.options.get_next_command([:list,:id])
+              case command
+              when :list
+                return {:type=>:object_list,:data=>api_shares_admin.read('data/users')[:data],:fields=>['username','email','directory_user','urn']}
+              when :id
+                res_id=self.options.get_next_argument('user id')
+                command=self.options.get_next_command([:app_authorizations,:authorize_share])
+                case command
+                when :app_authorizations
+                  return {:type=>:single_object,:data=>api_shares_admin.read("data/users/#{res_id}/app_authorizations")[:data]}
+                when :share
+                  share_name=self.options.get_next_argument('share name')
+                  all_shares=api_shares_admin.read('data/shares')[:data]
+                  share_id=all_shares.select{|s| s['name'].eql?(share_name)}.first['id']
+                  return {:type=>:single_object,:data=>api_shares_admin.create("data/shares/#{share_id}/user_permissions")[:data]}
+                end
+              end
+            when :share
+              command=self.options.get_next_command([:list,:name])
+              all_shares=api_shares_admin.read('data/shares')[:data]
+              case command
+              when :list
+                return {:type=>:object_list,:data=>all_shares,:fields=>['id','name','status','status_message']}
+              when :name
+                share_name=self.options.get_next_argument('share name')
+                share_id=all_shares.select{|s| s['name'].eql?(share_name)}.first['id']
+                raise "TODO"
+              end
+            end
+          end
+        end # execute action
+      end # Shares
+    end # Plugins
+  end # Cli
+end # Aspera

data/lib/aspera/cli/plugins/shares2.rb

@@ -0,0 +1,114 @@
+require 'aspera/cli/plugins/node'
+require 'uri'
+
+module Aspera
+  module Cli
+    module Plugins
+      class Shares2 < BasicAuthPlugin
+        def initialize(env)
+          super(env)
+          self.options.add_opt_simple(:organization,"organization")
+          self.options.add_opt_simple(:project,"project")
+          self.options.add_opt_simple(:share,"share")
+          self.options.parse_options!
+          return if env[:man_only]
+          unless env[:man_only]
+            # get parameters
+            shares2_api_base_url=self.options.get_option(:url,:mandatory)
+            shares2_username=self.options.get_option(:username,:mandatory)
+            shares2_password=self.options.get_option(:password,:mandatory)
+
+            # create object for REST calls to Shares2
+            @api_shares2_oauth=Rest.new({
+              :base_url => shares2_api_base_url,
+              :auth     => {
+              :type      => :oauth2,
+              :base_url  => shares2_api_base_url+'/oauth2',
+              :grant     => :header_userpass,
+              :user_name => shares2_username,
+              :user_pass => shares2_password
+              }})
+
+            @api_node=Rest.new({
+              :base_url => shares2_api_base_url+'/node_api',
+              :auth     => {
+              :type     => :basic,
+              :username => shares2_username,
+              :password => shares2_password}})
+          end
+        end
+
+        # path_prefix is either "" or "res/id/"
+        # adds : prefix+"res/id/"
+        # modify parameter string
+        def set_resource_path_by_id_or_name(resource_path,resource_sym)
+          res_id=self.options.get_option(resource_sym,:mandatory)
+          # lets get the class path
+          resource_path<<resource_sym.to_s+'s'
+          # is this an integer ? or a name
+          if res_id.to_i.to_s != res_id
+            all=@api_shares2_oauth.read(resource_path)[:data]
+            one=all.select{|i|i['name'].start_with?(res_id)}
+            Log.log.debug(one)
+            raise CliBadArgument,"No matching name for #{res_id} in #{all}" if one.empty?
+            raise CliBadArgument,"More than one match: #{one}" if one.length > 1
+            res_id=one.first['id'].to_s
+          end
+          Log.log.debug("res_id=#{res_id}")
+          resource_path<<'/'+res_id+'/'
+          return resource_path
+        end
+
+        # path_prefix is empty or ends with slash
+        def process_entity_action(resource_sym,path_prefix)
+          resource_path=path_prefix+resource_sym.to_s+'s'
+          operations=[:list,:create,:delete]
+          command=self.options.get_next_command(operations)
+          case command
+          when :create
+            params=self.options.get_next_argument("creation data (json structure)")
+            resp=@api_shares2_oauth.create(resource_path,params)
+            return {:data=>resp[:data],:type => :other_struct}
+          when :list
+            default_fields=['id','name']
+            query=self.options.get_option(:query,:optional)
+            args=query.nil? ? nil : {'json_query'=>query}
+            Log.log.debug("#{args}".bg_red)
+            return {:data=>@api_shares2_oauth.read(resource_path,args)[:data],:fields=>default_fields,:type=>:object_list}
+          when :delete
+            @api_shares2_oauth.delete(set_resource_path_by_id_or_name(path_prefix,resource_sym))
+            return Main.result_status('deleted')
+          when :info
+            return {:type=>:other_struct,:data=>@api_shares2_oauth.read(set_resource_path_by_id_or_name(path_prefix,resource_sym),args)[:data]}
+          else raise :ERROR
+          end
+        end
+
+        ACTIONS=[ :repository,:organization,:project,:team,:share,:appinfo,:userinfo,:admin]
+
+        def execute_action
+          command=self.options.get_next_command(ACTIONS)
+          case command
+          when :repository
+            command=self.options.get_next_command(Node::COMMON_ACTIONS)
+            return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: @api_node)).execute_action(command)
+          when :appinfo
+            node_info=@api_node.call({:operation=>'GET',:subpath=>'app',:headers=>{'Accept'=>'application/json','Content-Type'=>'application/json'}})[:data]
+            return { :type=>:single_object ,:data => node_info }
+          when :userinfo
+            node_info=@api_node.call({:operation=>'GET',:subpath=>'current_user',:headers=>{'Accept'=>'application/json','Content-Type'=>'application/json'}})[:data]
+            return { :type=>:single_object ,:data => node_info }
+          when :organization,:project,:share,:team
+            prefix=''
+            set_resource_path_by_id_or_name(prefix,:organization) if [:project,:team,:share].include?(command)
+            set_resource_path_by_id_or_name(prefix,:project) if [:share].include?(command)
+            process_entity_action(command,prefix)
+          when :admin
+            command=self.options.get_next_command([:users,:groups,:nodes])
+            return self.entity_action(@api_shares2_oauth,"system/#{command}",nil,:id)
+          end # command
+        end # execute_action
+      end # Files
+    end # Plugins
+  end # Cli
+end # Aspera

data/lib/aspera/cli/plugins/sync.rb

@@ -0,0 +1,65 @@
+require 'aspera/cli/plugin'
+require 'aspera/sync'
+require 'aspera/log'
+require 'open3'
+
+module Aspera
+  module Cli
+    module Plugins
+      # list and download connect client versions, select FASP implementation
+      class Sync < Plugin
+        def initialize(env)
+          super(env)
+          self.options.add_opt_simple(:parameters,"extended value for session set definition")
+          self.options.add_opt_simple(:session_name,"name of session to use for admin commands, by default first one")
+          self.options.parse_options!
+        end
+
+        ACTIONS=[ :start, :admin ]
+
+        def execute_action
+          command=self.options.get_next_command(ACTIONS)
+          case command
+          when :start
+            env_args=Aspera::Sync.new(self.options.get_option(:parameters,:mandatory)).compute_args
+            async_bin='async'
+            Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=\"#{v}\""}.join(' ')} \"#{async_bin}\" \"#{env_args[:args].join('" "')}\"")
+            res=system(env_args[:env],[async_bin,async_bin],*env_args[:args])
+            Log.log.debug("result=#{res}")
+            case res
+            when true; return Main.result_success
+            when false; raise "failed: #{$?}"
+            when nil; return Main.result_status("not started: #{$?}")
+            else raise "internal error: unspecified case"
+            end
+          when :admin
+            p=self.options.get_option(:parameters,:mandatory)
+            n=self.options.get_option(:session_name,:optional)
+            cmdline=['asyncadmin','--quiet']
+            if n.nil?
+              session=p['sessions'].first
+            else
+              session=p['sessions'].select{|s|s['name'].eql?(n)}.first
+            end
+            cmdline.push('--name='+session['name'])
+            if session.has_key?('local_db_dir')
+              cmdline.push('--local-db-dir='+session['local_db_dir'])
+            else
+              cmdline.push('--local-dir='+session['local_dir'])
+            end
+            command2=self.options.get_next_command([:status])
+            case command2
+            when :status
+              stdout, stderr, status = Open3.capture3(*cmdline)
+              Log.log.debug("status=#{status}, stderr=#{stderr}")
+              items=stdout.split("\n").inject({}){|m,l|i=l.split(/:  */);m[i.first.lstrip]=i.last.lstrip;m}
+              return {:type=>:single_object,:data=>items}
+            else raise "error"
+            end # command
+          else raise "error"
+          end # command
+        end # execute_action
+      end # Sync
+    end # Plugins
+  end # Cli
+end # Aspera