dtk-client 0.5.10 → 0.5.12

data/lib/commands/thor/component_module.rb

@@ -1,4 +1,3 @@
-dtk_require_from_base('command_helpers/ssh_processing')
 dtk_require_common_commands('thor/clone')
 dtk_require_common_commands('thor/list_diffs')
 dtk_require_common_commands('thor/push_to_remote')
@@ -7,6 +6,7 @@ dtk_require_common_commands('thor/push_clone_changes')
 dtk_require_common_commands('thor/edit')
 dtk_require_common_commands('thor/reparse')
 dtk_require_common_commands('thor/purge_clone')
+dtk_require_common_commands('thor/common')
 dtk_require_from_base('configurator')
 dtk_require_from_base('command_helpers/service_importer')
 
@@ -127,6 +127,12 @@ module DTK::Client
       version = options.version
       component_module_name = get_module_name(component_module_id)
 
+      # if called from import-git first param will be git_url and second one will be component_module_id
+      git_import = context_params.get_forwarded_options()[:git_import] if context_params.get_forwarded_options()
+      if git_import
+        component_module_id = context_params.retrieve_arguments([:option_2!],method_argument_names)
+      end
+
       unless (options.force? || method_opts[:force_delete])
         # Ask user if really want to delete component module and all items contained in it, if not then return to dtk-shell without deleting
         return unless Console.confirmation_prompt("Are you sure you want to delete component-module #{version.nil? ? '' : 'version '}'#{component_module_name}#{version.nil? ? '' : ('-' + version.to_s)}' and all items contained in it"+'?')
@@ -215,9 +221,11 @@ TODO: might deprecate
         return module_info_about(context_params, :components, :component)
       end
 
-      action           = (options.remote? ? "list_remote" : "list")
-#      post_body        = (options.remote? ? { :rsa_pub_key => SshProcessing.rsa_pub_key_content() } : {:detail_to_include => ["remotes","versions"]})
-      post_body        = (options.remote? ? { :rsa_pub_key => SshProcessing.rsa_pub_key_content() } : {:detail_to_include => ["remotes"]})
+      forwarded_remote = context_params.get_forwarded_options()["remote"] if context_params.get_forwarded_options()
+      remote           = options.remote? || forwarded_remote
+      action           = (remote ? "list_remote" : "list")
+#     post_body        = (options.remote? ? { :rsa_pub_key => SSHUtil.rsa_pub_key_content() } : {:detail_to_include => ["remotes","versions"]})
+      post_body        = (remote ? { :rsa_pub_key => SSHUtil.rsa_pub_key_content() } : {:detail_to_include => ["remotes"]})
       post_body[:diff] = options.diff? ? options.diff : {}
       response         = post rest_url("component_module/#{action}"),post_body
       
@@ -232,7 +240,7 @@ TODO: might deprecate
       post_body = {
         :component_module_id => component_module_id,
         :detail_to_include => ["remotes"],
-        :rsa_pub_key => SshProcessing.rsa_pub_key_content()
+        :rsa_pub_key => SSHUtil.rsa_pub_key_content()
       }
       response = post rest_url("component_module/versions"), post_body
 
@@ -374,7 +382,7 @@ TODO: might deprecate
       post_body = {
         :remote_module_name => remote_module_name,
         :local_module_name => local_module_name,
-        :rsa_pub_key => SshProcessing.rsa_pub_key_content()
+        :rsa_pub_key => SSHUtil.rsa_pub_key_content()
       }
       post_body.merge!(:do_not_raise => do_not_raise) if do_not_raise
       post_body.merge!(:ignore_component_error => ignore_component_error) if ignore_component_error
@@ -420,7 +428,6 @@ TODO: might deprecate
       context_params.forward_options({:git_import => true})
       # Reuse module create method to create module from local component_module
       create_response = import(context_params)
-
        
       if create_response.ok?
         if external_dependencies = create_response.data(:external_dependencies)
@@ -495,7 +502,7 @@ TODO: might deprecate
       post_body = {
        :remote_module_name      => remote_name,
        :remote_module_namespace => remote_namespace,
-       :rsa_pub_key => SshProcessing.rsa_pub_key_content()
+       :rsa_pub_key => SSHUtil.rsa_pub_key_content()
       }
       post rest_url("component_module/delete_remote"), post_body
     end
@@ -508,7 +515,7 @@ TODO: might deprecate
     #   post_body = {
     #     :component_module_id => component_module_id,
     #     :remote_component_name => input_remote_name,
-    #     :rsa_pub_key => SshProcessing.rsa_pub_key_content()
+    #     :rsa_pub_key => SSHUtil.rsa_pub_key_content()
     #   }
 
     #   response = post rest_url("component_module/export"), post_body
@@ -523,7 +530,7 @@ TODO: might deprecate
       post_body = {
         :component_module_id => component_module_id,
         :remote_component_name => input_remote_name,
-        :rsa_pub_key => SshProcessing.rsa_pub_key_content()
+        :rsa_pub_key => SSHUtil.rsa_pub_key_content()
       }
 
       response = post rest_url("component_module/export"), post_body
@@ -571,39 +578,22 @@ TODO: might deprecate
 #       push_to_remote_aux(:component_module, component_module_id, component_module_name, options["namespace"], version)
 #     end
 
-#    desc "COMPONENT-MODULE-NAME/ID pull-from-dtkn [-v VERSION]", "Update local component module from remote repository."
-    # version_method_option
-    # desc "COMPONENT-MODULE-NAME/ID pull-from-dtkn", "Update local component module from remote repository."
-    # def pull_from_dtkn(context_params)     
-    #   component_module_id, component_module_name = context_params.retrieve_arguments([:component_module_id!,:component_module_name],method_argument_names)
-    #   version = options["version"]
-
-    #   response = pull_from_remote_aux(:component_module,component_module_id,version)
-    #   return response unless response.ok?
-
-    #   if component_module_name.to_s =~ /^[0-9]+$/
-    #     component_module_id = component_module_name
-    #     component_module_name = get_module_name(component_module_id)
-    #   end
-
-    #   modules_path    = OsUtil.module_clone_location()
-    #   module_location = "#{modules_path}/#{component_module_name}#{version && "-#{version}"}"
-
-    #   push_clone_changes_aux(:component_module,component_module_id,version,nil,true) if File.directory?(module_location)
-    #   Response::Ok.new()
-    # end
+#   desc "COMPONENT-MODULE-NAME/ID pull-from-dtkn [-n NAMESPACE] [-v VERSION]", "Update local component module from remote repository."
+    desc "COMPONENT-MODULE-NAME/ID pull-dtkn [-n NAMESPACE]", "Update local component module from remote repository."
+    method_option "namespace",:aliases => "-n",
+      :type => :string, 
+      :banner => "NAMESPACE",
+      :desc => "Remote namespace"
 
-    desc "COMPONENT-MODULE-NAME/ID pull-dtkn", "Update local component module from remote repository."
     def pull_dtkn(context_params)     
-#      component_module_id, component_module_name, catalog = context_params.retrieve_arguments([:component_module_id!,:component_module_name,:option_1],method_argument_names)
       component_module_id, component_module_name = context_params.retrieve_arguments([:component_module_id!,:component_module_name,:option_1],method_argument_names)
       catalog = 'dtkn'
       version = options["version"]
-      
       raise DtkValidationError, "You have to provide valid catalog to pull changes from! Valid catalogs: #{PullCatalogs}" unless catalog
 
       if catalog.to_s.eql?("dtkn")
-        response = pull_from_remote_aux(:component_module,component_module_id,version)
+        opts = {:version => version, :remote_namespace => options.namespace}
+        response = pull_from_remote_aux(:component_module,component_module_id,opts)
         return response unless response.ok?
 
         if component_module_name.to_s =~ /^[0-9]+$/
@@ -634,7 +624,7 @@ TODO: might deprecate
 
       post_body = {
         :component_module_id => component_module_id,
-        :rsa_pub_key => SshProcessing.rsa_pub_key_content()
+        :rsa_pub_key => SSHUtil.rsa_pub_key_content()
       }
       response = post rest_url("component_module/versions"), post_body
       return response unless response.ok?
@@ -770,23 +760,21 @@ TODO: might deprecate
 #      if catalog.to_s.eql?("origin")
 #        push_clone_changes_aux(:component_module,component_module_id,version,options["message"]||DEFAULT_COMMIT_MSG,internal_trigger)
       if catalog.to_s.eql?("dtkn")
+        remote_module_info = get_remote_module_info_aux(:component_module, component_module_id, options["namespace"], version)
+        return remote_module_info unless remote_module_info.ok?
+
         unless File.directory?(module_location)
-          if Console.confirmation_prompt("Unable to push to remote because module '#{component_module_name}#{version && "-#{version}"}' has not been cloned. Would you like to clone module now"+'?')
-            response = clone_aux(:component_module,component_module_id,version,false)
-          
-            if(response.nil? || response.ok?)
-              reparse_aux(module_location)
-              push_to_remote_aux(:component_module, component_module_id, component_module_name, options["namespace"], version)  if Console.confirmation_prompt("Would you like to push changes to remote"+'?')
-            end
+          response = clone_aux(:component_module, component_module_id, version, true, true)
 
-            return response
-          else
-            # user choose not to clone needed module
-            return
+          if(response.nil? || response.ok?)
+            reparse_aux(module_location)
+            response = push_to_remote_aux(remote_module_info, :component_module)
           end
+
+          return response
         end
 
-        push_to_remote_aux(:component_module, component_module_id, component_module_name, options["namespace"], version)
+        push_to_remote_aux(remote_module_info, :component_module)
       else
         raise DtkValidationError, "You have to provide valid catalog to push changes to! Valid catalogs: #{PushCatalogs}"
       end

data/lib/commands/thor/developer.rb

@@ -15,7 +15,7 @@ module DTK::Client
 
       ids = []
       # get all nodes which id starts with node_pattern
-      nodes["data"].collect{|a| ids<<a["id"].to_i if a["id"].to_s.start_with?(node_pattern.to_s)}
+      nodes["data"].collect{|a| ids<<a["id"].to_i if a["id"].to_s.match(Regexp.new(node_pattern.to_s)) }
       raise DTK::Client::DtkValidationError, "Unable to find nodes to match this pattern: '#{node_pattern}'." if ids.empty?
 
       # if it doesn't contain extension upload both *.rb and *.ddl

data/lib/commands/thor/node.rb

@@ -15,7 +15,8 @@ module DTK::Client
     end
 
     def self.valid_children()
-      [:component, :utils]
+      # [:component, :utils]
+      [:utils]
     end
 
     def self.multi_context_children()
@@ -85,32 +86,26 @@ module DTK::Client
        post rest_url("node/info"), post_body
     end
 
-    desc "NODE-NAME/ID ssh [--keypair] [--remote-user]","SSH into node, optional parameters are path to keypair and remote user."
+    desc "NODE-NAME/ID ssh REMOTE-USER [--keypair PATH-TO-PEM]","SSH into node, optional parameters are path to keypair and remote user."
     method_option "--keypair",:type => :string, :desc => "Keypair used for connection, if not provided default is used", :banner => "KEYPAIR"
-    method_option "--remote-user",:type => :string, :desc => "Remote user used for connection", :banner => "REMOTE USER"
     def ssh(context_params)
       if OsUtil.is_windows?
         puts "[NOTICE] SSH functionality is currenly not supported on Windows."
         return
       end
 
-      node_id = context_params.retrieve_arguments([:node_id!],method_argument_names)
-
-
-      keypair_location = options.keypair || OsUtil.dtk_keypair_location()
+      node_id, remote_user = context_params.retrieve_arguments([:node_id!,:option_1!],method_argument_names)
+      keypair_location = options.keypair
 
-
-      remote_user = options.send('remote-user') || 'ubuntu'
-
-      unless File.exists?(keypair_location||'')
-        error_message = keypair_location ? "Not able to find keypair, '#{keypair_location}'" : "Default keypair not set, please provide one in 'ssh' command"
-        raise ::DTK::Client::DtkError, error_message
+      if keypair_location
+        unless File.exists?(keypair_location)
+          raise ::DTK::Client::DtkError, "Not able to find keypair, '#{keypair_location}'"
+        end
       end
 
       context_params.forward_options({ :json_return => true })
       response = info_aux(context_params)
 
-
       if response.ok?
         node_info = response.data['nodes'].find { |n| node_id == n['node_id'] }
         public_dns = node_info ? node_info['external_ref']['ec2_public_address'] : nil
@@ -118,7 +113,28 @@ module DTK::Client
         raise ::DTK::Client::DtkError, "Not able to resolve instance address, has instance been stopped?" unless public_dns
 
         connection_string = "#{remote_user}@#{public_dns}"
-        ssh_command = "ssh  -o \"StrictHostKeyChecking no\" -o \"UserKnownHostsFile /dev/null\" -i #{keypair_location} #{connection_string}"
+
+        default_keypair = OsUtil.dtk_keypair_location()
+
+        # vanilla ssh command using only pub key
+        vanilla_ssh = 
+
+        ssh_command = nil
+        
+        if keypair_location
+          # provided PEM key
+          ssh_command = "ssh -o \"StrictHostKeyChecking no\" -o \"UserKnownHostsFile /dev/null\" -i #{keypair_location} #{connection_string}"
+        elsif SSHUtil.ssh_reachable?(remote_user, public_dns)
+          # it has PUB key access
+          ssh_command = "ssh -o \"StrictHostKeyChecking no\" -o \"UserKnownHostsFile /dev/null\" #{connection_string}"
+        else
+          # using default keypair
+          if default_keypair
+            ssh_command = "ssh -o \"StrictHostKeyChecking no\" -o \"UserKnownHostsFile /dev/null\" -i #{default_keypair} #{connection_string}"
+          end
+        end
+
+        raise ::DTK::Client::DtkError, "No public key access or PEM provided, please grant access or provide valid PEM key" if ssh_command.nil?
 
         OsUtil.print("You are entering SSH terminal (#{connection_string}) ...", :yellow)
         Kernel.system(ssh_command)

data/lib/commands/thor/node_template.rb

@@ -5,10 +5,6 @@ module DTK::Client
       PPColumns.get(:node)
     end
 
-    def self.whoami()
-      return :node_template, "node/list", {:subtype => 'template'}
-    end
-
     def self.extended_context()
       {
         :context => {
@@ -16,6 +12,10 @@ module DTK::Client
         }
       }
     end
+
+    def self.validation_list(context_params)
+      get_cached_response(:node_template, "node/list", {:subtype => 'template'})
+    end
 =begin
   #Not implemented yet
     desc "NODE-TEMPLATE-NAME/ID info", "Get information about given node template."

data/lib/commands/thor/provider.rb

@@ -1,10 +1,6 @@
 module DTK::Client
   class Provider < CommandBaseThor
 
-    def self.whoami()
-      return :provider, "target/list", { :subtype => :template }
-    end
-
     def self.valid_children()
       [:target]
     end
@@ -13,6 +9,10 @@ module DTK::Client
       [:target]
     end
 
+    def self.validation_list(context_params)
+      get_cached_response(:provider, "target/list", {:subtype => :template })
+    end
+
     def self.override_allowed_methods()
       return DTK::Shell::OverrideTasks.new({
         :command_only => {

data/lib/commands/thor/service.rb

@@ -10,6 +10,7 @@ dtk_require_common_commands('thor/set_required_params')
 dtk_require_common_commands('thor/edit')
 dtk_require_common_commands('thor/purge_clone')
 dtk_require_common_commands('thor/assembly_workspace')
+dtk_require_common_commands('thor/action_result_handler')
 
 LOG_SLEEP_TIME   = DTK::Configuration.get(:tail_log_frequency)
 DEBUG_SLEEP_TIME = DTK::Configuration.get(:debug_task_frequency)
@@ -23,6 +24,7 @@ module DTK::Client
       include EditMixin
       include PurgeCloneMixin
       include AssemblyWorkspaceMixin
+      include ActionResultHandler
 
       def get_assembly_name(assembly_id)
         get_name_from_id_helper(assembly_id)
@@ -42,6 +44,8 @@ module DTK::Client
         raise DtkError,"[ERROR] Illegal name (#{assembly_name}) for service." unless assembly_id
         assembly_id
       end
+
+
       
     end
 
@@ -158,7 +162,7 @@ TODO: overlaps with different meaning
             ['set-attribute',"set-attribute ATTRIBUTE-NAME [VALUE] [-u]","# (Un)Set attribute value. The option -u will unset the attribute's value."],
             ['start', "start", "# Start node instance."],
             ['stop', "stop", "# Stop node instance."],
-            ['ssh', "ssh [--keypair] [--remote-user]", "# SSH into node, optional parameters are path to keypair and remote user."]
+            ['ssh', "ssh REMOTE-USER [--keypair PATH-TO-PEM]", "# SSH into node, optional parameters are path to keypair and remote user."]
           ],
           :component => [
             ['info',"info","# Return info about component instance belonging to given node."],
@@ -197,36 +201,6 @@ TODO: overlaps with different meaning
     def cancel_task(context_params)
       cancel_task_aux(context_params)
     end
-=begin
-#TODO: until we investigate DTK-1349
-    desc "rename SERVICE-NAME NEW-SERVICE-NAME","Change service name."
-    def rename(context_params)
-      assembly_name, new_assembly_name = context_params.retrieve_arguments([:option_1!,:option_2!],method_argument_names)
-
-      if assembly_name.to_s =~ /^[0-9]+$/
-        assembly_id = assembly_name
-      else
-        assembly_id = get_assembly_id(assembly_name)
-      end
-
-      post_body = {
-        :assembly_id => assembly_id,
-        :assembly_name => assembly_name,
-        :new_assembly_name => new_assembly_name
-      }
-
-      response = post rest_url("assembly/rename"), post_body
-      return response unless response.ok?
-
-      @@invalidate_map << :service
-      response      
-    end
-=end
-
-    #desc "ASSEMBLY-NAME/ID clear-tasks", "Clears the tasks that have been run already."
-    #def clear_tasks(context_params)
-    #  clear_tasks_aux(context_params)
-    #end
 
     desc "SERVICE-NAME/ID create-assembly SERVICE-MODULE-NAME ASSEMBLY-NAME", "Create a new assembly from this service instance in the designated service module."
     def create_assembly(context_params)
@@ -340,21 +314,6 @@ TODO: will put in dot release and will rename to 'extend'
       task_status_aw_aux(context_params)
     end
 
-=begin
-    desc "ASSEMBLY-NAME/ID run-smoketests", "Run smoketests associated with assembly instance"
-    def run_smoketests(context_params)
-      assembly_id = context_params.retrieve_arguments([:assembly_id!],method_argument_names)
-      post_body = {
-        :assembly_id => assembly_id
-      }
-      # create smoke test
-      response = post rest_url("assembly/create_smoketests_task"), post_body
-      return response unless response.ok?
-      # execute
-      task_id = response.data(:task_id)
-      post rest_url("task/execute"), "task_id" => task_id
-    end
-=end
 
     desc "SERVICE-NAME/ID list-nodes","List nodes associated with service."
     def list_nodes(context_params)
@@ -480,6 +439,63 @@ TODO: will put in dot release and will rename to 'extend'
     #  post rest_url("assembly/list_smoketests"), post_body
     #end
 
+    desc "SERVICE-NAME/ID grant-access SYS-USER NAME [PATH-TO-PUB-KEY]", "Grant access to given service and its nodes" 
+    def grant_access(context_params)
+      service_id, system_user, rsa_key_name, path_to_rsa_pub_key = context_params.retrieve_arguments([:service_id!,:option_1!, :option_2!, :option_3],method_argument_names)
+
+      path_to_rsa_pub_key ||= SSHUtil.default_rsa_pub_key_path()
+      rsa_pub_key_content = SSHUtil.read_and_validate_pub_key(path_to_rsa_pub_key)
+    
+      response = post_file rest_url("assembly/initiate_ssh_pub_access"), {
+        :agent_action => :grant_access,
+        :system_user => system_user, 
+        :rsa_pub_name => rsa_key_name, 
+        :rsa_pub_key => rsa_pub_key_content, 
+        :assembly_id => service_id 
+      }
+
+      return response unless response.ok?
+
+      action_results_id = response.data(:action_results_id)
+
+      print_action_results(action_results_id)
+
+      nil
+    end
+
+    desc "SERVICE-NAME/ID revoke-access SYS-USER NAME", "Revoke access to given service and its nodes" 
+    def revoke_access(context_params)
+      service_id, system_user, rsa_key_name = context_params.retrieve_arguments([:service_id!,:option_1!, :option_2!],method_argument_names)
+
+      response = post_file rest_url("assembly/initiate_ssh_pub_access"), { 
+        :agent_action => :revoke_access,
+        :system_user => system_user, 
+        :rsa_pub_name => rsa_key_name,
+        :assembly_id => service_id 
+      }
+
+      return response unless response.ok?
+
+      action_results_id = response.data(:action_results_id)
+
+      print_action_results(action_results_id)
+
+      nil
+    end
+
+    desc "SERVICE-NAME/ID list-ssh-access", "List SSH access for each of the nodes" 
+    def list_ssh_access(context_params)
+      service_id = context_params.retrieve_arguments([:service_id!],method_argument_names)
+
+      response = post_file rest_url("assembly/list_ssh_access"), { 
+        :assembly_id => service_id 
+      }
+
+      response.render_table(:ssh_access)
+      response
+    end
+
+
     desc "SERVICE-NAME/ID info", "Get info about content of the service."
     def info(context_params)
       info_aux(context_params)
@@ -500,16 +516,15 @@ TODO: will put in dot release and will rename to 'extend'
       else
         assembly_id = get_assembly_id(assembly_name)
       end
-      # assembly_name = get_assembly_name(assembly_id)
 
       unless options.force?
         # Ask user if really want to delete assembly, if not then return to dtk-shell without deleting
-        #used form "+'?' because ?" confused emacs ruby rendering
+        # used form "+'?' because ?" confused emacs ruby rendering
         what = "service"
         return unless Console.confirmation_prompt("Are you sure you want to delete and destroy #{what} '#{assembly_name}' and its nodes"+'?')
       end
 
-      #purge local clone
+      # purge local clone
       response = purge_clone_aux(:all,:assembly_module => {:assembly_name => assembly_name})
       return response unless response.ok?
 
@@ -540,21 +555,6 @@ TODO: will put in dot release and will rename to 'extend'
       create_attribute_aux(context_params)
     end
 
-  #  desc "ASSEMBLY-NAME/ID add-assembly ASSEMBLY-TEMPLATE-NAME/ID", "Add (stage) an assembly template to become part of this assembly instance"
-  #  method_option "auto-complete",:aliases => "-a" ,
-  #    :type => :boolean, 
-  #    :default=> false,
-  #    :desc => "Automatically add in connections"
-  #  def add_assembly(context_params)
-  #    assembly_id,assembly_template_id = context_params.retrieve_arguments([:assembly_id,:option_1!],method_argument_names)
-  #    post_body = {
-  #      :assembly_id => assembly_id,
-  #      :assembly_template_id => assembly_template_id
-  #    }
-  #    post_body.merge!(:auto_add_connections => true) if options.auto_complete?
-  #    post rest_url("assembly/add_assembly_template"), post_body
-  #  end
-
     # using ^^ before NODE-NAME to remove this command from assembly/assembly_id/node/node_id but show in assembly/assembly_id
     desc "SERVICE-NAME/ID create-node ^^NODE-NAME NODE-TEMPLATE", "Add (stage) a new node in the service."
     def create_node(context_params)