dtk-shell 0.10.0

data/lib/command_helpers/service_importer.rb

@@ -0,0 +1,251 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+dtk_require_from_base('configurator')
+module DTK::Client
+  #
+  # Main purpose of this module is to recognize which local modules are missing based on
+  # name, namespace, version and for those missing component module module will call
+  # module#clone and module#import_dtkn method to get missing component modules
+  #
+  module ServiceImporter
+    def create_missing_clone_dirs()
+      Configurator.create_missing_clone_dirs
+    end
+
+    def self.error_message(name, errors, opts = {})
+      prefix = ''
+      unless opts[:module_type] == :service_module
+        prefix = "Module '#{name}' has errors:\n  "
+      end
+      command = opts[:command] || 'edit'
+      "#{prefix}#{errors.to_s}\nYou can fix errors by invoking the '#{command}' command.\n"
+    end
+
+    ##
+    # Method will trigger pull from dtkn for each existing module
+    #
+    def trigger_module_auto_pull(required_modules, opts = {})
+      return if required_modules.empty?
+      hide_output = opts[:hide_output]
+
+      # options[:force] means this command is triggered from trigger_module_auto_import method bellow
+      unless opts[:force]
+        update_none = RemoteDependencyUtil.check_for_frozen_modules(required_modules)
+
+        if update_none
+          print "All dependent modules are frozen and will not be updated!\n" unless hide_output
+          print "Resuming pull ... " unless hide_output
+          return Response::Ok.new()
+        end
+      end
+
+      if opts[:force] || Console.confirmation_prompt("Do you want to update in addition to this module its dependent modules from the catalog?")
+        required_modules.uniq.each do |r_module|
+          module_name = full_module_name(r_module)
+          module_type = r_module['type']
+          version     = r_module['version']
+          full_name   = (version && !version.eql?('master')) ? "#{module_name}(#{version})" : module_name
+
+          # No op for frozen modules
+          next if r_module['frozen']
+
+          print "Pulling #{module_type.gsub('_',' ')} content for '#{full_name}' ... " unless hide_output
+
+          new_context_params = DTK::Shell::ContextParams.new
+          new_context_params.add_context_to_params(module_type, module_type)
+          new_context_params.add_context_name_to_params(module_type, module_type, module_name)
+
+          forwarded_opts = { :skip_recursive_pull => true, :ignore_dependency_merge_conflict => true }
+          forwarded_opts.merge!(:do_not_raise => true) if opts[:do_not_raise]
+          forwarded_opts.merge!(:version => version) if version && !version.eql?('master')
+          new_context_params.forward_options(forwarded_opts)
+
+          response = ContextRouter.routeTask(module_type, "pull_dtkn", new_context_params, @conn)
+
+          unless response.ok?
+            if opts[:do_not_raise]
+              OsUtil.print("#{response.error_message}", :red)
+            else
+              raise DtkError, response.error_message
+            end
+          end
+        end
+
+        print "Resuming pull ... " unless opts[:force]
+      end
+    end
+
+    ##
+    # Method will trigger import for each missing module component
+    #
+    def trigger_module_auto_import(modules_to_import, required_modules, opts = {})
+      hide_output = opts[:hide_output]
+
+      puts 'Auto-installing missing module(s)' unless hide_output
+      update_all  = false
+      update_none = RemoteDependencyUtil.check_for_frozen_modules(required_modules)
+
+      # Print out or update installed modules from catalog
+      required_modules.uniq.each do |r_module|
+        module_name         = full_module_name(r_module)
+        module_type         = r_module['type']
+        version             = r_module['version']
+        full_name           = (version && !version.eql?('master')) ? "#{module_name}(#{version})" : module_name
+
+        unless hide_output
+          print "Using #{module_type.gsub('_', ' ')} '#{full_name}'\n" unless update_all
+        end
+
+        next if update_none || opts[:update_none]
+
+        pull_opts = {:force => true, :do_not_raise => true}
+        pull_opts.merge!(:hide_output => hide_output) if hide_output
+
+        if update_all
+          trigger_module_auto_pull([r_module], pull_opts)
+        else
+          options = required_modules.size > 1 ? %w(all none) : []
+          update  = Console.confirmation_prompt_additional_options("Do you want to update dependent #{module_type.gsub('_', ' ')} '#{full_name}' from the catalog?", options) unless hide_output
+          next unless update
+
+          if update.to_s.eql?('all')
+            update_all = true
+            trigger_module_auto_pull([r_module], pull_opts)
+          elsif update.to_s.eql?('none')
+            update_none = true
+          else
+            trigger_module_auto_pull([r_module], pull_opts)
+          end
+        end
+      end
+
+      # Trigger import/install for missing modules
+      modules_to_import.uniq.each do |m_module|
+        module_name = full_module_name(m_module)
+        module_type = m_module['type']
+        version     = m_module['version']
+        full_name   = (version && !version.eql?('master')) ? "#{module_name}(#{version})" : module_name
+
+        # we check if there is module_url if so we install from git
+        module_url  = m_module['module_url']
+
+        # descriptive message
+        importing  = module_url ? "Importing" : "Installing"
+        import_msg = "#{importing} #{module_type.gsub('_', ' ')} '#{full_name}'"
+        import_msg += " from git source #{module_url}" if module_url
+        print "#{import_msg} ... " unless hide_output
+
+        if module_url
+          # import from Git source
+          new_context_params = ::DTK::Shell::ContextParams.new([module_url, module_name])
+          new_context_params.forward_options(:internal_trigger => true)
+          response = ContextRouter.routeTask(module_type, 'import_git', new_context_params, @conn)
+        else
+          # import from Repo Manager
+          new_context_params = ::DTK::Shell::ContextParams.new([module_name])
+          new_context_params.override_method_argument!('option_2', version) if version && !version.eql?('master')
+          new_context_params.forward_options(:skip_cloning => false, :skip_auto_install => true, :module_type => module_type, :ignore_component_error => true).merge!(opts)
+          response = ContextRouter.routeTask(module_type, 'install', new_context_params, @conn)
+        end
+
+        ignore_component_error = (new_context_params.get_forwarded_options() || {})[:ignore_component_error] && module_type.eql?('component_module')
+        puts(response.data(:does_not_exist) ? response.data(:does_not_exist) : 'Done.') unless hide_output
+        raise DtkError, response.error_message if !response.ok? && !ignore_component_error
+      end
+
+      Response::Ok.new()
+    end
+
+    def resolve_missing_components(service_module_id, service_module_name, namespace_to_use, force_clone=false)
+      # Get dependency component modules and cross reference them with local component modules
+      module_component_list = post rest_url("service_module/list_component_modules"), { :service_module_id => service_module_id }
+
+      local_modules, needed_modules = OsUtil.local_component_module_list(), Array.new
+
+      if module_component_list
+        module_component_list.data.each do |cmp_module|
+          with_namespace = ModuleUtil.resolve_name(cmp_module["display_name"],cmp_module["namespace_name"])
+          formated_name = add_version?(with_namespace, cmp_module['version'])
+          unless local_modules.include?(formated_name)
+            needed_modules << cmp_module.merge({'formated_name' => formated_name})
+          end
+        end
+      end
+
+      unless needed_modules.empty?
+        # puts "Service '#{service_module_name}' does not have the following component modules dependencies on the client machine: \n\n"
+        # needed_modules.each { |m| puts " - #{m['formated_name']}" }
+        is_install_dependencies = true
+        # is_install_dependencies = Console.confirmation_prompt("\nDo you want to clone these missing component modules to the client machine?") unless force_clone
+
+        # we get list of modules available on server
+
+        new_context_params = nil
+
+        if is_install_dependencies
+          needed_modules.each do |m|
+            formated_name = m['formated_name']
+            # print "Cloning component module '#{formated_name}' from server ... "
+            thor_options = {}
+            thor_options["version"] = m['version'] unless m['version'].eql?('base')
+            thor_options["skip_edit"] = true
+            thor_options["omit_output"] = true
+            thor_options.merge!(:module_type => 'component-module')
+            thor_options.merge!(:service_importer => true)
+            new_context_params = ::DTK::Shell::ContextParams.new
+            new_context_params.forward_options(thor_options)
+            new_context_params.add_context_to_params(formated_name, :"component-module", m['id'])
+
+            begin
+              response = ContextRouter.routeTask("component_module", "clone", new_context_params, @conn)
+            rescue DtkValidationError => e
+              # ignoring this
+            end
+            # puts "Done."
+          end
+        end
+      end
+    end
+
+    private
+    #
+    # As the result we can have multiple version so we need to resolve them
+    #
+    # Returns: Array<String>
+    def resolve_module_names(e)
+      versions = (e['version'] ? e['version'].split(',') : ['CURRENT'])
+
+      versions.collect { |version| add_version?(e['display_name'], version)}
+    end
+
+    # Resolves local module name
+    #
+    # Returns: String
+    def add_version?(display_name, version)
+      version = nil if 'CURRENT'.eql?(version) || 'base'.eql?(version)
+      (version ? "#{display_name}-#{version.strip}" : "#{display_name}")
+    end
+
+  private
+
+    def full_module_name(module_hash)
+      ModuleUtil.join_name(module_hash['name'], module_hash['namespace'])
+    end
+
+  end
+end

data/lib/command_helpers/service_link.rb

@@ -0,0 +1,33 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK; module Client; class CommandHelper
+  class ServiceLink < self; class << self
+    def post_body_with_id_keys(context_params,method_argument_names)
+      assembly_or_workspace_id = context_params.retrieve_arguments([[:service_id!,:workspace_id!]])
+      ret = {:assembly_id => assembly_or_workspace_id}
+      if context_params.is_last_command_eql_to?(:component)
+        component_id,service_type = context_params.retrieve_arguments([:component_id!,:option_1!],method_argument_names)
+        ret.merge(:input_component_id => component_id,:service_type => service_type)
+      else
+        service_link_id = context_params.retrieve_arguments([:option_1!],method_argument_names)
+        ret.merge(:service_link_id => service_link_id)
+      end
+    end
+
+  end; end
+end; end; end

data/lib/command_helpers/test_module_creator.rb

@@ -0,0 +1,69 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'fileutils'
+require 'erubis'
+
+module DTK::Client
+	class TestModuleCreator
+		class << self
+			def create_clone(type, module_name)
+				Response.wrap_helper_actions do
+		      modules_dir = OsUtil.test_clone_location()
+		      FileUtils.mkdir_p(modules_dir) unless File.directory?(modules_dir)
+		      target_repo_dir = OsUtil.module_location(type,module_name)
+
+		      begin
+		      	FileUtils.mkdir_p(target_repo_dir)
+		        generate_model(module_name, target_repo_dir)
+		        generate_serverspec_files(module_name, target_repo_dir)
+		      rescue => e
+		      	additional_error_msg = ""
+		        error_msg = "Create of directory (#{target_repo_dir}) failed."
+		        additional_error_msg = "Directory already exists" if e.message.include? "File exists"
+		        raise DTK::ErrorUsage.new(error_msg + " " + additional_error_msg,:log_error=>false)
+		      end
+		      {"module_directory" => target_repo_dir}
+		    end
+			end
+
+			def generate_model(module_name, target_repo_dir)
+				input = File.expand_path('test_module_templates/dtk.model.yaml.eruby', File.dirname(__FILE__))
+				eruby = Erubis::Eruby.new(File.read(input))
+				content = eruby.result(:module_name => module_name)
+				File.open(target_repo_dir + "/dtk.model.yaml", "w") { |f| f.write(content) }
+			end
+
+			def generate_serverspec_files(module_name, target_repo_dir)
+				template_location = File.expand_path('test_module_templates', File.dirname(__FILE__))
+				spec_helper_template = Erubis::Eruby.new(File.read(template_location + "/spec_helper.rb.eruby")).result
+				spec_template = Erubis::Eruby.new(File.read(template_location + "/temp_component_spec.rb.eruby")).result
+
+				begin
+					#Create standard serverspec structure
+					FileUtils.mkdir_p(target_repo_dir + "/serverspec/spec/localhost")
+					File.open(target_repo_dir + "/serverspec/spec/spec_helper.rb", "w") { |f| f.write(spec_helper_template) }
+					File.open(target_repo_dir + "/serverspec/spec/localhost/temp_component_spec.rb", "w") { |f| f.write(spec_template) }
+				rescue => e
+					error_msg = "Generating serverspec files failed."
+		      DtkLogger.instance.error_pp(e.message, e.backtrace)
+		      raise DTK::ErrorUsage.new(error_msg,:log_error=>false)
+				end
+			end
+		end
+	end
+end

data/lib/command_helpers/test_module_templates/dtk.model.yaml.eruby

@@ -0,0 +1,10 @@
+module: <%= module_name %>
+dsl_version: 0.9.1
+module_type: serverspec
+components:
+  temp_component:
+    external_ref:
+      serverspec_test: temp_component_spec.rb
+    attributes:
+      temp_attribute:
+        type: string

data/lib/command_helpers/test_module_templates/spec_helper.rb.eruby

@@ -0,0 +1,10 @@
+require 'serverspec'
+
+include SpecInfra::Helper::Exec
+include SpecInfra::Helper::DetectOS
+
+class DtkAttribute
+	def DtkAttribute.get(name)
+		Thread.current[name.to_sym]
+	end
+end

data/lib/command_helpers/test_module_templates/temp_component_spec.rb.eruby

@@ -0,0 +1,5 @@
+require File.expand_path('../spec_helper', File.dirname(__FILE__))
+
+# Gets attribute defined in dtk.model.yaml
+# This attribute value can be used further in spec examples
+attribute = DtkAttribute.get(:temp_attribute)

data/lib/commands.rb

@@ -0,0 +1,57 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK
+  module Client
+    module CommandBase
+      #TODO: temp workaround
+      def rotate_args(rotated_args)
+        [rotated_args.last] + rotated_args[0..rotated_args.size-2]
+      end
+
+      def get(url)
+        get_connection.get(self.class,url)
+      end
+      def post(url,body=nil)
+        get_connection.post(self.class,url,body)
+      end
+
+      def post_file(url,body=nil)
+        get_connection.post_file(self.class,url,body)
+      end
+
+      def rest_url(route)
+        get_connection.rest_url(route)
+      end
+
+      def get_connection
+        DTK::Client::Session.get_connection()
+      end
+
+      def self.handle_argument_error(task, error)
+        super
+      end
+
+     private
+
+      def pretty_print_cols()
+        self.class.pretty_print_cols()
+      end
+    end
+
+  end
+end

data/lib/commands/common/thor/access_control.rb

@@ -0,0 +1,133 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::Client
+  module AccessControlMixin
+
+    def chmod_aux(module_id, permission_string, namespace = nil, chmod_action = :chmod)
+      permission_selector = PermissionUtil.validate_permissions!(permission_string.downcase)
+      post_body = {
+        :module_id => module_id,
+        :permission_selector => permission_selector,
+        :chmod_action        => chmod_action,
+        :rsa_pub_key         => SSHUtil.rsa_pub_key_content(),
+        :remote_module_namespace => namespace
+      }
+      response = post rest_url("#{resolve_module_type}/remote_chmod"), post_body
+      return response unless response.ok?
+
+      if response.data(:simple_flow)
+        puts "Module is now public."
+      else
+        # in case there are dependencies
+        main_module_name = response.data(:main_module)['full_name']
+        puts "Main module '#{main_module_name}' has dependencies that are not public: "
+        unless response.data(:missing_modules).empty?
+          missing = response.data(:missing_modules).collect { |a| a['full_name'] }
+          OsUtil.print("  These modules are missing on repository: #{missing.join(', ')}", :red)
+        end
+        unless response.data(:no_permission).empty?
+          no_permission = response.data(:no_permission).collect { |a| a['full_name'] }
+          OsUtil.print("  You cannot change permissions for dependencies: #{no_permission.join(', ')}", :yellow)
+        end
+        unless response.data(:with_permission).empty?
+          with_permission = response.data(:with_permission)
+          with_permission_names = with_permission.collect { |a| a['full_name'] }
+          OsUtil.print("  You can change permissions for dependencies: #{with_permission_names.join(', ')}", :white)
+
+          # fix for bug in comments for DTK-1959
+          # need to send hash instead of array to be able to parse properly in rest_request_params
+          with_permission_hash = {}
+          with_permission.each do |wp|
+            with_permission_hash.merge!("#{wp['name']}" => wp)
+          end
+
+          response.data["with_permission"] = with_permission_hash
+        end
+
+        puts "How should we resolve these dependencies: "
+        input = Shell::InteractiveWizard.text_input("(A)ll / (M)ain Module / (N)one ", true, /M|A|N/i)
+        if 'N'.eql?(input)
+          return nil
+        else
+          puts "Sending input information ... "
+          post_body = {
+            :module_id     => module_id,
+            :module_info   => response.data,
+            :public_action => 'A'.eql?(input) ? :all : :one,
+            :rsa_pub_key   => SSHUtil.rsa_pub_key_content(),
+            :remote_module_namespace => namespace
+          }
+
+          response = post rest_url("#{resolve_module_type}/confirm_make_public"), post_body
+          return response unless response.ok?
+          puts "Modules are now public."
+        end
+      end
+
+      nil
+    end
+
+    def chown_aux(module_id, remote_user, namespace = nil)
+      post_body = {
+        :module_id => module_id,
+        :remote_user => remote_user,
+        :rsa_pub_key         => SSHUtil.rsa_pub_key_content(),
+        :remote_module_namespace => namespace
+      }
+      post rest_url("#{resolve_module_type}/remote_chown"), post_body
+    end
+
+    def collaboration_aux(action, module_id, users, groups, namespace = nil)
+      raise DtkValidationError, "You must provide --users or --groups to this command" if users.nil? && groups.nil?
+      post_body = {
+        :module_id => module_id,
+        :users  => users,
+        :groups => groups,
+        :action => action,
+        :remote_module_namespace => namespace,
+        :rsa_pub_key => SSHUtil.rsa_pub_key_content()
+      }
+      post rest_url("#{resolve_module_type}/remote_collaboration"), post_body
+    end
+
+    def collaboration_list_aux(module_id, namespace = nil)
+      post_body = {
+        :module_id => module_id,
+        :remote_module_namespace => namespace,
+        :rsa_pub_key => SSHUtil.rsa_pub_key_content()
+      }
+      post rest_url("#{resolve_module_type}/list_remote_collaboration"), post_body
+    end
+
+  private
+
+    def resolve_module_type
+      case self
+        when ComponentModule
+          return :component_module
+        when ServiceModule
+          return :service_module
+        when TestModule
+          return :test_module
+        else
+          raise DtkError, "Module type cannot be resolved for this class (#{self})"
+        end
+    end
+
+  end
+end