dtk-shell 0.10.0

data/lib/commands/common/thor/module/import.rb

@@ -0,0 +1,210 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+dtk_require_from_base("util/os_util")
+dtk_require_from_base('commands')
+dtk_require_from_base("command_helper")
+
+module DTK::Client
+  class CommonModule
+    class Import < BaseCommandHelper
+      include CommandBase
+      include CommandHelperMixin
+      include PushCloneChangesMixin
+      include ReparseMixin
+
+      def from_git(internal_trigger = false)
+        OsUtil.print('Retrieving git module data, please wait ...') unless internal_trigger
+
+        git_repo_url, module_name    = retrieve_arguments([:option_1!, :option_2!])
+        namespace, local_module_name = get_namespace_and_name(module_name, ModuleUtil::NAMESPACE_SEPERATOR)
+
+        module_type  = @command.get_module_type(@context_params)
+        thor_options = { :git_import => true}
+
+        unless namespace
+          namespace_response = post rest_url("namespace/default_namespace_name")
+          return namespace_response unless namespace_response.ok?
+
+          namespace = namespace_response.data
+          thor_options[:default_namespace] = namespace
+        end
+
+        opts = {
+          :namespace => namespace,
+          :branch    => @options['branch']
+        }
+
+        response = Helper(:git_repo).create_clone_from_optional_branch(module_type.to_sym, local_module_name, git_repo_url, opts)
+        return response unless response.ok?
+
+        # Remove .git directory to rid of git pointing to user's github
+        FileUtils.rm_rf("#{response['data']['module_directory']}/.git")
+
+        @context_params.forward_options(thor_options)
+        create_response = from_git_or_file()
+
+        unless create_response.ok?
+          delete_dir        = namespace.nil? ? local_module_name : "#{namespace}/#{local_module_name}"
+          full_module_name  = create_response.data[:full_module_name]
+          local_module_name = full_module_name.nil? ? delete_dir : full_module_name
+
+          @command.delete_module_sub_aux(@context_params, local_module_name, :force_delete => true, :no_error_msg => true, :purge => true)
+          return create_response
+        end
+
+        opts_pull = {
+          :local_branch => @branch,
+          :namespace    => @module_namespace
+        }
+        pull_response = Helper(:git_repo).pull_changes(module_type, @module_name, opts_pull)
+        return pull_response unless pull_response.ok?
+
+        if external_dependencies = create_response.data(:external_dependencies)
+          print_external_dependencies(external_dependencies, 'in the git repo')
+        end
+
+        unless internal_trigger
+          OsUtil.print("Successfully installed #{ModuleUtil.module_name(module_type)} '#{ModuleUtil.join_name(@module_name, @module_namespace)}' from git.", :green)
+        end
+      end
+
+      def from_file()
+        module_type = @command.get_module_type(@context_params)
+        module_name = retrieve_arguments([:option_1!])
+        opts        = {}
+        namespace, local_module_name = get_namespace_and_name(module_name, ModuleUtil::NAMESPACE_SEPERATOR)
+
+        response = from_git_or_file()
+        return response unless response.ok?
+
+        opts_pull = {
+          :local_branch => @branch,
+          :namespace    => @module_namespace
+        }
+        resp = Helper(:git_repo).pull_changes(module_type, @module_name, opts_pull)
+        return resp unless resp.ok?
+
+        if error = response.data(:dsl_parse_error)
+          dsl_parsed_message = ServiceImporter.error_message(module_name, error)
+          DTK::Client::OsUtil.print(dsl_parsed_message, :red)
+        end
+
+        # remove source directory if no errors while importing
+        module_final_dir = @repo_obj.repo_dir
+        if @old_dir and (@old_dir != module_final_dir)
+          FileUtils.rm_rf(@old_dir) unless namespace
+        end
+
+        if external_dependencies = response.data(:external_dependencies)
+          print_external_dependencies(external_dependencies, 'dtk.model.yaml includes')
+        end
+
+        # if user do import from default directory (e.g. import ntp - without namespace) print message
+        DTK::Client::OsUtil.print("Module '#{@new_module_name}' has been created and module directory moved to #{module_final_dir}",:yellow) unless namespace
+
+        Response::Ok.new()
+      end
+
+
+      private
+
+      def from_git_or_file()
+
+        default_ns = @context_params.get_forwarded_options()[:default_namespace]
+        git_import = @context_params.get_forwarded_options()[:git_import]
+
+        name_option = git_import ? :option_2! : :option_1!
+
+        if git_import
+          module_git_url, module_name = @context_params.retrieve_arguments([:option_1!, :option_2!])
+        else
+          module_name, module_git_url = @context_params.retrieve_arguments([:option_1!, :option_2!])
+        end
+
+        module_type = @command.get_module_type(@context_params)
+        version     = @options["version"]
+
+        # extract namespace and module_name from full name (r8:maven will result in namespace = r8 & name = maven)
+        namespace, local_module_name = get_namespace_and_name(module_name, ModuleUtil::NAMESPACE_SEPERATOR)
+        namespace = default_ns if default_ns && namespace.nil?
+
+        # first check that there is a directory there and it is not already a git repo, and it ha appropriate content
+        response = Helper(:git_repo).check_local_dir_exists_with_content(module_type.to_sym, local_module_name, nil, namespace)
+        return response unless response.ok?
+
+        #check for yaml/json parsing errors before import
+        module_directory = response.data(:module_directory)
+        reparse_aux(module_directory)
+
+        # first make call to server to create an empty repo
+        post_body = {
+          :module_name => local_module_name,
+          :module_namespace => namespace,
+          :module_git_url   => module_git_url
+        }
+
+        response = post(rest_url("#{module_type}/create"), post_body)
+        return response unless response.ok?
+
+        repo_url, repo_id, @module_id, branch, @new_module_name = response.data(:repo_url, :repo_id, :module_id, :workspace_branch, :full_module_name)
+        response = Helper(:git_repo).rename_and_initialize_clone_and_push(module_type.to_sym, local_module_name, @new_module_name, branch, repo_url, module_directory)
+        return response unless (response && response.ok?)
+
+        @repo_obj, commit_sha = response.data(:repo_obj, :commit_sha)
+        module_final_dir      = @repo_obj.repo_dir
+        @old_dir              = response.data[:old_dir]
+
+        post_body = {
+          :repo_id    => repo_id,
+          :commit_sha => commit_sha,
+          :commit_dsl => true,
+          :scaffold_if_no_dsl => true,
+          "#{module_type}_id".to_sym => @module_id
+        }
+
+        if git_import
+          post_body.merge!(:git_import => true)
+        else
+          post_body.merge!(:update_from_includes => true)
+        end
+
+        response = post(rest_url("#{module_type}/update_from_initial_create"), post_body)
+
+        unless response.ok?
+          response.set_data_hash({ :full_module_name => @new_module_name })
+          # remove new directory and leave the old one if import without namespace failed
+          if @old_dir and (@old_dir != module_final_dir)
+            FileUtils.rm_rf(module_final_dir) unless (namespace && git_import)
+          end
+          return response
+        end
+
+        dsl_updated_info = response.data(:dsl_updated_info)
+        dsl_created_info = response.data(:dsl_created_info)
+        DTK::Client::OsUtil.print("A module_refs.yaml file has been created for you, located at #{module_final_dir}", :yellow) if dsl_updated_info && !dsl_updated_info.empty?
+        DTK::Client::OsUtil.print("A #{dsl_created_info["path"]} file has been created for you, located at #{module_final_dir}", :yellow) if dsl_created_info && !dsl_created_info.empty?
+
+        @module_name, @module_namespace, repo_url, @branch, not_ok_response = workspace_branch_info(module_type, @module_id, version)
+        return not_ok_response if not_ok_response
+
+        response
+      end
+
+    end
+  end
+end

data/lib/commands/common/thor/node.rb

@@ -0,0 +1,53 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::Client
+  module NodeMixin
+    def get_node_info_for_ssh_login(node_id, context_params)
+      context_params.forward_options(:json_return => true)
+      response = info_aux(context_params)
+      return response unless response.ok?
+      # Should only have info about the specfic node_id
+      
+      unless node_info = response.data(:nodes).find{ |node| node_id == (node['node_properties'] || {})['node_id'] }
+        raise DtkError, "Cannot find info about node with id '#{node_id}'"
+      end
+
+      data = {}
+      node_properties = node_info['node_properties'] || {}
+      if public_dns = node_properties['ec2_public_address']
+        data.merge!('public_dns' => public_dns)
+      end
+      if default_login_user = NodeMixin.default_login_user?(node_properties)
+        data.merge!('default_login_user' => default_login_user)
+      end
+
+      Response::Ok.new(data)
+    end
+
+    def self.default_login_user?(node_properties)
+    if os_type = node_properties['os_type']
+        DefaultLoginByOSType[os_type]
+      end
+    end
+
+    DefaultLoginByOSType = {
+      'ubuntu'       => 'ubuntu',
+      'amazon-linux' => 'ec2-user'
+    }
+  end
+end

data/lib/commands/common/thor/poller.rb

@@ -0,0 +1,65 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::Client
+  module Poller
+
+    PERIOD_WAIT_TIME = 1
+
+    def poller_response(wait_time = PERIOD_WAIT_TIME)
+      begin
+        response = nil
+        thread   = Thread.new(self) do |main_thread|
+          begin
+            while true
+              messages_response = main_thread.get main_thread.rest_url("messages/retrieve")
+              print_response(messages_response.data) if messages_response.ok?
+              sleep(wait_time)
+            end
+          rescue => e
+            puts e.message
+            pp e.backtrace
+          end
+        end
+
+        response = yield
+      ensure
+        thread.kill
+      end
+      response
+    end
+
+    def print_response(message_array)
+      message_array.each do |msg|
+        DTK::Client::OsUtil.print(msg['message'], resolve_type(msg['type']))
+      end
+    end
+
+    def resolve_type(message_type)
+      case message_type.to_sym
+      when :info
+        :white
+      when :warning
+        :yellow
+      when :error
+        :red
+      else
+        :white
+      end
+    end
+  end
+end

data/lib/commands/common/thor/pull_clone_changes.rb

@@ -0,0 +1,28 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+dtk_require_common_commands('thor/common')
+module DTK::Client
+  module PullCloneChangesMixin
+    def pull_clone_changes?(module_type,module_id,version=nil,opts={})
+      module_name, module_namespace,repo_url,branch,not_ok_response = workspace_branch_info(module_type,module_id,version,opts)
+      return not_ok_response if not_ok_response
+      opts_pull = opts.merge(:local_branch => branch,:namespace => module_namespace)
+      Helper(:git_repo).pull_changes(module_type,module_name,opts_pull)
+    end
+  end
+end

data/lib/commands/common/thor/pull_from_remote.rb

@@ -0,0 +1,152 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::Client
+  module PullFromRemoteMixin
+    def pull_from_remote_aux(module_type,module_id,opts={})
+      version = opts[:version]
+      remote_namespace = opts[:remote_namespace]
+      #get remote module info, errors raised if remote is not linked or access errors
+      path_to_key = SSHUtil.default_rsa_pub_key_path()
+      rsa_pub_key = File.file?(path_to_key) && File.open(path_to_key){|f|f.read}.chomp
+
+      post_body = PostBody.new(
+        PullFromRemote.id_field(module_type) => module_id,
+        :access_rights => "r",
+        :action => "pull",
+        :version? => version,
+        :remote_namespace? => remote_namespace,
+        :rsa_pub_key? => rsa_pub_key
+      )
+      response = post(rest_url("#{module_type}/get_remote_module_info"),post_body)
+      return response unless response.ok?
+
+      module_name,full_module_name, frozen = response.data(:module_name, :full_module_name, :frozen)
+      raise DtkError, "You are not allowed to update frozen #{module_type} versions!" if frozen
+
+      remote_params = response.data_hash_form(:remote_repo_url,:remote_repo,:remote_branch)
+      remote_params.merge!(:version => version) if version
+
+      # check and import component module dependencies before importing service itself
+      unless opts[:skip_recursive_pull]
+        import_module_component_dependencies(module_type, module_id, remote_namespace)
+      end
+
+      # check whether a local module exists to determine whether pull from local clone or try to pull from server
+      # TODO: probably remove OsUtil.print("Pulling changes from remote: #{remote_params[:remote_repo]} @ #{remote_params[:remote_repo_url]}")
+
+      if Helper(:git_repo).local_clone_dir_exists?(module_type, module_name, :full_module_name => full_module_name, :version => version)
+        unless rsa_pub_key
+          raise DtkError,"No File found at (#{path_to_key}). Path is wrong or it is necessary to generate the public rsa key (e.g., run ssh-keygen -t rsa)"
+        end
+        opts_perform_locally = remote_params.merge(
+          :full_module_name => full_module_name,
+          :force => opts[:force],
+          :do_not_raise => opts[:do_not_raise],
+          :ignore_dependency_merge_conflict => opts[:ignore_dependency_merge_conflict]
+        )
+        PullFromRemote.perform_locally(self,module_type,module_id,module_name,opts_perform_locally)
+      else
+        # TODO: see if this works correctly
+        PullFromRemote.perform_on_server(self,module_type,module_id,full_module_name,remote_params)
+      end
+    end
+
+   private
+
+    ##
+    #
+    # module_type: will be :component_module or :service_module
+    def import_module_component_dependencies(module_type, module_id, remote_namespace = nil)
+      response = resolve_pull_from_remote_on_server(module_type, module_id, remote_namespace)
+
+      print "Resolving dependencies please wait ... "
+      RemoteDependencyUtil.check_permission_warnings(response)
+
+      # install them all!
+      if (response.ok? && !(missing_components = response.data(:missing_modules)).empty?)
+        required_modules = response.data(:required_modules)
+        puts " New dependencies found, Installing."
+
+        trigger_module_auto_import(missing_components, required_modules, { :include_pull_action => true })
+
+        puts "Resuming pull from remote ..."
+      else
+        puts 'Done.'
+      end
+
+      # pull them all!
+      if (response.ok? && !(required_modules = response.data(:required_modules)).empty?)
+        trigger_module_auto_pull(required_modules)
+      end
+
+      RemoteDependencyUtil.print_dependency_warnings(response)
+      nil
+    end
+
+  private
+
+    def resolve_pull_from_remote_on_server(module_type, module_id, remote_namespace=nil)
+      post_body = PostBody.new(
+        :module_id => module_id,
+        :remote_namespace? => remote_namespace,
+        :rsa_pub_key => SSHUtil.rsa_pub_key_content()
+      )
+      post(rest_url("#{module_type}/resolve_pull_from_remote"),post_body)
+    end
+
+    module PullFromRemote
+      extend CommandBase
+      def self.perform_locally(cmd_obj,module_type,module_id,module_name,remote_params)
+        opts = remote_params
+        response = cmd_obj.Helper(:git_repo).pull_changes(module_type,module_name,opts)
+
+        # return response unless response.ok?
+        if custom_message = response.data[:custom_message]
+          puts custom_message
+        elsif (response.data[:diffs].nil? || response.data[:diffs].empty?)
+          puts "No changes to pull from remote.".colorize(:yellow) unless response['errors']
+        else
+          puts "Changes pulled from remote".colorize(:green)
+        end
+
+        response
+      end
+
+      def self.perform_on_server(cmd_obj,module_type,module_id,module_name,remote_params)
+        #TODO: this does not handle different namespaces; so suggesting workaround for now
+        raise DtkError, "Module must be cloned to perform this operation; execute 'clone' command and then retry."
+        post_body = {
+          id_field(module_type) => module_id,
+          :remote_repo => remote_params[:remote_repo],
+          :module_name => module_name
+        }
+        post_body.merge!(:version => remote_params[:version]) if remote_params[:version]
+        response = post rest_url("#{module_type}/pull_from_remote"), post_body
+
+
+        puts "You have successfully pulled code on server instance." if response.ok?
+        response
+      end
+
+      def self.id_field(module_type)
+        "#{module_type}_id"
+      end
+
+    end
+  end
+end