dtk-client 0.10.5 → 0.10.6

data/lib/client/operation/module/push_dtkn/convert_source.rb

@@ -54,21 +54,24 @@ module DTK::Client; class Operation::Module
 
         create_and_checkout_branch?(current_branch, target_repo_dir, "remotes/dtkn/master") do |repo|
           FileUtils.mkdir_p("#{target_repo_dir}/assemblies") unless File.exists?("#{target_repo_dir}/assemblies")
-          
+
           args = [transform_helper, ServiceInfo.info_type, service_info['remote_repo_url'], parent]
           service_file_path__content_array.each { |file| Operation::ClientModuleDir.create_file_with_content("#{service_file_path(target_repo_dir, file, *args)}", file[:content]) }
-          
+
           commit_and_push_to_remote(repo, target_repo_dir, "master", "dtkn")
         end
       end
 
       def self.transform_component_info(target_repo_dir, parent, component_info, parsed_common_module, current_branch)
-        transform_helper = ServiceAndComponentInfo::TransformTo.new(target_repo_dir, parent.module_ref, parent.version, parsed_common_module)
+        transform_helper                   = ServiceAndComponentInfo::TransformTo.new(target_repo_dir, parent.module_ref, parent.version, parsed_common_module)
         component_file_path__content_array = ComponentInfo.transform_info(transform_helper, component_info['remote_repo_url'], parent)
+        component_info_remote              = "remotes/dtkn-component-info/master"
 
-        create_and_checkout_branch?(current_branch, target_repo_dir, "remotes/dtkn-component-info/master") do |repo|
-          component_file_path__content_array.each { |file| Operation::ClientModuleDir.create_file_with_content("#{file_path(target_repo_dir, file)}", file[:content]) }
+        create_and_checkout_branch?(current_branch, target_repo_dir, component_info_remote) do |repo|
+          # find deleted files from master branch and delete them in component info cached branch ("remotes/dtkn-component-info/master")
+          delete_diffs(repo, component_info_remote, current_branch, target_repo_dir)
 
+          component_file_path__content_array.each { |file| Operation::ClientModuleDir.create_file_with_content("#{file_path(target_repo_dir, file)}", file[:content]) }
           commit_and_push_to_remote(repo, target_repo_dir, "master", "dtkn-component-info")
         end
       end
@@ -77,6 +80,16 @@ module DTK::Client; class Operation::Module
 
       attr_reader :info_processor, :target_repo_dir, :parent
 
+      def self.delete_diffs(repo, component_info_remote, current_branch, target_repo_dir)
+        if diffs = repo.diff_name_status(component_info_remote, current_branch)
+          deletes_and_renames = diffs.select{ |k,v| v.eql?('D') || v.start_with?('R') }
+          unless deletes_and_renames.empty?
+            to_delete = deletes_and_renames.keys.select { |key| !key.include?('dtk.model.yaml') && !key.include?('module_refs.yaml') }
+            to_delete.each { |file| Operation::ClientModuleDir.rm_f("#{target_repo_dir}/#{file}") }
+          end
+        end
+      end
+
       def self.write_output_path_text_pairs(transform_helper, target_repo_dir, info_types_processed)
       end
 

data/lib/client/operation/module/push_dtkn.rb

@@ -52,7 +52,7 @@ module DTK::Client
         end
 
         error_msg = "To allow push-dtkn to go through, invoke 'dtk push' to push the changes to server before invoking push-dtkn again"
-        GitRepo.modified_with_diff?(@target_repo_dir, { :error_msg => error_msg })
+        GitRepo.modified_with_diff?(@target_repo_dir, { :error_msg => error_msg, :command => 'push-dtkn' })
 
         query_string_hash = QueryStringHash.new(
           :module_name => @module_ref.module_name,

data/lib/client/operation/module/stage.rb

@@ -36,7 +36,7 @@ module DTK::Client
           )
           
           error_msg = "To allow stage to go through, invoke 'dtk push' to push the changes to server before invoking stage again"
-          GitRepo.modified_with_diff?(directory_path || module_ref.client_dir_path, { :error_msg => error_msg }) unless force
+          GitRepo.modified_with_diff?(directory_path || module_ref.client_dir_path, { :error_msg => error_msg, :command => 'stage' }) unless force
           service_name ||= rest_post("#{BaseRoute}/generate_service_name", post_body).data
           base_path = ClientModuleDir.ret_base_path(:service, service_name)
           

data/lib/client/operation/module/uninstall.rb

@@ -23,8 +23,7 @@ module DTK::Client
           module_ref  = args.required(:module_ref)
           name        = args.required(:name)
           version     = args.required(:version)
-          versions    = nil
-
+  
           unless name.nil?
             query_string_hash = QueryStringHash.new(
               :detail_to_include => ['remotes', 'versions']
@@ -32,44 +31,69 @@ module DTK::Client
             response = rest_get("#{BaseRoute}/list", query_string_hash)
             installed_modules = response.data
 
-            name.gsub!("/", ":")
-            installed_modules.each do |module_val| 
-              if module_val["display_name"].eql? name
-                val = name.split(":")
-                if version.nil?
-                  versions = module_val["versions"].split(",").map(&:strip) 
-                  if versions.size > 1
-                    version = Console.version_prompt(versions, "Select which module version to uninstall: ", { :add_all => true})
-                    module_val["versions"] if version.eql? "all"
-                  else
-                    version = module_val["versions"]
-                  end
-                end
-
-                module_opts = {
-                  :module_name => val[1],
-                  :namespace   => val[0],
-                  :version     => version
-                }
-                module_ref = ModuleRef.new(module_opts)
-              end
-            end
+            module_ref = process_module_ref(installed_modules, name, version) 
           end 
-          
+
           raise Error::Usage, "Invalid module name." if module_ref.nil?
 
+          delete_versions = module_ref.version
+
           unless args[:skip_prompt]
-            return false unless Console.prompt_yes_no("Are you sure you want to uninstall module '#{module_ref.pretty_print}' from the DTK Server?", :add_options => true)
+            if delete_versions && delete_versions.is_a?(Array) 
+               return false unless Console.prompt_yes_no("Are you sure you want to uninstall all module versions for '#{module_ref.namespace}/#{module_ref.module_name}' from the DTK Server?", :add_options => true) 
+            else
+              return false unless Console.prompt_yes_no("Are you sure you want to uninstall module '#{module_ref.pretty_print}' from the DTK Server?", :add_options => true) 
+            end
+            
           end
 
           post_body = module_ref_post_body(module_ref)
-          post_body.merge!(:versions => versions) if versions
-
           rest_post("#{BaseRoute}/delete", post_body)
-          OsUtil.print_info("DTK module '#{module_ref.pretty_print}' has been uninstalled successfully.")
+
+          error_msg =
+            if delete_versions && delete_versions.is_a?(Array) && delete_versions.size > 1
+              "All versions of dtk module '#{module_ref.namespace}/#{module_ref.module_name}' have been uninstalled."
+            else
+              "DTK module '#{module_ref.pretty_print}' has been uninstalled successfully."
+            end
+
+          OsUtil.print_info(error_msg)
           nil
         end
       end
+      
+        def self.process_module_ref(installed_modules, name, version)
+          name.gsub!('/', ':')
+          module_ref = nil
+
+          installed_modules.each do |module_val| 
+            if module_val["display_name"].eql? name
+              val = name.split(":")
+              if version.nil?
+                versions = module_val["versions"].split(",").map(&:strip) 
+                versions.each { |value| value.tr!('*', '') }
+
+                if versions.size > 1
+                  version = Console.version_prompt(versions, "Select which module version to uninstall: ", { :add_all => true})
+                  version = versions if version.eql? "all"
+                else
+                  version = module_val["versions"]
+                  version.tr!('*', '') if version.include?('*')
+                end
+              end
+
+              module_opts = {
+                :module_name => val[1],
+                :namespace   => val[0],
+                :version     => version
+              }
+              
+              module_ref = ModuleRef.new(module_opts) 
+            end
+          end
+
+          module_ref
+        end
 
     end
   end

data/lib/client/operation/service/converge.rb

@@ -29,7 +29,7 @@ module DTK::Client
           )
 
           error_msg = "To allow converge to go through, invoke 'dtk push' to push the changes to server before invoking converge again"
-          GitRepo.modified_with_diff?(module_dir, { :error_msg => error_msg }) unless force
+          GitRepo.modified_with_diff?(module_dir, { :error_msg => error_msg, :command => 'converge' }) unless force
           violations = find_violations(service_instance)
           return violations if violations
 

data/lib/client/operation/service/delete.rb

@@ -30,14 +30,25 @@ module DTK::Client
           end
 
           error_msg = "To allow delete to go through, invoke 'dtk push' to push the changes to server before invoking delete again"
-          GitRepo.modified_with_diff?(directory_path || @module_ref.client_dir_path, { :error_msg => error_msg }) unless force
+          GitRepo.modified_with_diff?(directory_path || @module_ref.client_dir_path, { :error_msg => error_msg, :command => 'delete' }) unless force
           post_body = PostBody.new(
             :service_instance => service_instance,
             :recursive? => recursive
           )
-          rest_post("#{BaseRoute}/delete", post_body)
+          response = rest_post("#{BaseRoute}/delete", post_body)
 
           OsUtil.print_info("Delete procedure started. For more information use 'dtk task-status'.")
+          display_node_info(response.data)
+        end
+      end
+
+      def self.display_node_info(nodes, message = '')
+        if nodes.size > 0
+          nodes.each do |node|
+            return if node['admin_op_status'] == 'pending' || node['external_ref']["instance_id"].nil?
+            message += "#{node["display_name"]} - #{ node["external_ref"]["instance_id"]}\n" unless node["display_name"].eql?("node") && node["dtk_client_type"].eql?("node_group") 
+          end
+          OsUtil.print("Nodes that will be deleted: \n" + message)
         end
       end
 

data/lib/client/operation/service/exec.rb

@@ -35,8 +35,8 @@ module DTK::Client
           end
           
           error_msg = "To allow #{args[:command]} to go through, invoke 'dtk push' to push the changes to server before invoking #{args[:command]} again"
-          GitRepo.modified_with_diff?(directory_path, { :error_msg => error_msg })
-
+          GitRepo.modified_with_diff?(directory_path, { :error_msg => error_msg, :command => 'exec'})
+        
           post_body = PostBody.new(
             :task_params? => task_params
           )

data/lib/client/operation/service/uninstall.rb

@@ -22,7 +22,9 @@ module DTK::Client
         wrap_operation(args) do |args|
           service_instance = args.required(:service_instance)
           recursive        = args.required(:recursive)
-          delete            = args.required(:delete)
+          delete           = args.required(:delete)
+          path             = args[:directory_path]
+          node             = []
 
           unless args[:skip_prompt]
             return false unless Console.prompt_yes_no("Are you sure you want to uninstall the infrastructure associated with '#{service_instance}' and delete this service instance from the server?", :add_options => true)
@@ -30,19 +32,20 @@ module DTK::Client
 
           post_body = PostBody.new(
             :service_instance => service_instance,
-            :recursive? => recursive,
+            :recursive?  => recursive,
             :delete      => delete
           )
-          rest_post("#{BaseRoute}/uninstall", post_body)
+          response = rest_post("#{BaseRoute}/uninstall", post_body)
 
-          ClientModuleDir.rm_f(args[:directory_path]) if args[:purge]
+          ClientModuleDir.rm_f(path) if args[:purge]
 
-          OsUtil.print_info("DTK module '#{service_instance}' has been uninstalled successfully.")
+          info = "DTK module '#{service_instance}' has been uninstalled successfully." 
+          OsUtil.print_info(info)
+          Delete.display_node_info(response.data) if delete
         end
       end
 
+
     end
   end
-end
-
-
+end

data/lib/client/render/view/table/processor.rb

@@ -45,6 +45,7 @@ module DTK::Client
         @evaluated_data = []
         @error_data     = []
         @action_data    = []
+        @footnote       = nil
       end
       private :initialize
 
@@ -61,6 +62,7 @@ module DTK::Client
         data = [data] unless data.kind_of?(Array)
 
         print_error_table = opts[:print_error_table]
+        @footnote         = opts[:footnote]
 
 
         data.each do |data_element|
@@ -158,6 +160,11 @@ module DTK::Client
             printed << message
           end
         end
+
+        if @footnote
+          printf " \n"
+          printf "%15s %s\n", "INFO:".colorize(:yellow), @footnote.colorize(:yellow)
+        end
       end
       
       

data/lib/client/render.rb

@@ -53,6 +53,7 @@ module DTK::Client
       if render_type == Type::TABLE
         render_opts = {
           :print_error_table => opts[:print_error_table],
+          :footnote          => opts[:footnote]
         }
         get_adapter(Type::TABLE, opts).render(ruby_obj, render_opts)
       elsif ruby_obj.kind_of?(Hash)

data/lib/client/response/render_helper.rb

@@ -25,6 +25,7 @@ module DTK::Client
         @semantic_datatype = nil
         @skip_render       = false
         @render_type       = Render::Type::DEFAULT
+        @footnote          = nil
       end
       private :render_attributes_init!
 
@@ -36,12 +37,13 @@ module DTK::Client
         render_opts = {
           :render_type       => @render_type,
           :semantic_datatype => @semantic_datatype,
-          :print_error_table => @print_error_table
+          :print_error_table => @print_error_table,
+          :footnote          => @footnote
         }
         Render.render(data, render_opts)
       end
           
-      def set_render_as_table!(semantic_datatype = nil)
+      def set_render_as_table!(semantic_datatype = nil, footnote = nil)
         return self unless ok?
 
         unless semantic_datatype ||= semantic_datatype_in_payload
@@ -53,6 +55,7 @@ module DTK::Client
         end
         @semantic_datatype = normalize_semantic_datatype(semantic_datatype)
         @render_type = Render::Type::TABLE
+        @footnote    = footnote
         self
       end
 

data/lib/client/service_and_component_info/transform_from/info/component.rb

@@ -44,7 +44,7 @@ module DTK::Client
 
       def component_dsl_path
         matches = directory_file_paths.select { |path| component_dsl_input_files_processor.match?(path) }
-        raise Error, "Unexpected that there is not a unique component dsl file" if matches.size != 1
+        raise Error::MissingDslFile, "Unexpected that there is not a unique component dsl file" if matches.size != 1
         matches.first
       end
 

data/lib/client/util/os_util.rb

@@ -96,6 +96,31 @@ module DTK::Client
       nil
     end
 
+    def self.gem_list
+      `gem list`.split("\n")
+    end
+
+    def self.gem_names_versions_hash
+      name_version_hash = {}
+      gem_list.each do |gem|
+        if gem_match = gem.match(/(^.*)\((.*)\)/)
+          name = gem_match[1].strip
+          versions = (gem_match[2]||'').split(',')
+          name_version_hash.merge!(name => versions)
+        end
+      end
+      name_version_hash
+    end
+
+    def self.gem_installed?(gem_name, gem_version = nil)
+      gems_hash = gem_names_versions_hash
+      if gem_version
+        (gems_hash[gem_name]||[]).include?(gem_version)
+      else
+        gems_hash.keys.include?(gem_name)
+      end
+    end
+
     private
     
     def self.genv(name)

data/lib/dtk_puppet/parse_structure.rb

@@ -0,0 +1,180 @@
+ module DTK::Puppet
+  class ParseStructure < Hash
+    def initialize(ast_item = nil, _opts = {})
+      return super() if ast_item.nil?
+    end
+
+    #### used in generate_meta
+    def config_agent_type
+      :puppet
+    end
+
+    def render_hash_form(module_name)
+      rendered_hash = {}
+
+      self[:children].each do |child|
+        name       = child['name'].gsub("#{module_name}::", '')
+        type       = child['type']
+        type_name  = child['name']
+        attributes = nil
+
+        attributes = process_attributes(child['attributes']) if child['attributes']
+        actions    = calculate_action(type, type_name)
+
+        child_name = name.gsub('::', '_')
+        child_hash = { child_name => {} }
+
+        child_hash[child_name].merge!('attributes' => attributes) if attributes
+        child_hash[child_name].merge!('actions' => actions)
+
+        rendered_hash.merge!(child_hash)
+      end
+
+      rendered_hash
+    end
+
+    def process_attributes(attributes)
+      processed_atttibutes = {}
+      attributes.each do |attr|
+        attr_name = attr['name']
+        attr_type = attr['type']
+        attr_hash = { attr_name => { 'type' => attr_type } }
+
+        if required = attr['required']
+          attr_hash[attr_name].merge!('required' => required)
+        end
+
+        processed_atttibutes.merge!(attr_hash)
+      end
+
+      processed_atttibutes
+    end
+
+    def remove_quotations(str)
+      if str.starts_with?('"')
+        str = str.slice(1..-1)
+      end
+      if str.ends_with?('"')
+        str = str.slice(0..-2)
+      end
+    end 
+
+    def calculate_action(type, type_name)
+      { 'create' => { "puppet_#{type}" => type_name } }
+    end
+
+    def string_to_boolean(string)
+      if string.to_s == 'true'
+        true
+      elsif string.to_s == 'false'
+        false
+      end
+    end
+
+    def self.create(ast_obj, opts = {})
+      new(ast_obj, opts)
+    end
+
+    TPS = [:hostclass, :definition]
+    class TopPS < self
+      def initialize(ast_array = nil, opts = {})
+        self[:children] = []
+        add_children(ast_array, opts)
+        super
+      end
+
+      def add_children(ast_array, opts = {})
+        return unless ast_array
+
+        ast_array.instantiate('').each do |ast_item|
+          ast_item_type = ast_item.type
+          child =
+            if TPS.include?(ast_item_type)
+              ComponentPS.create(ast_item, opts)
+            else
+              fail DTK::Client::Error.new("Unexpected top level ast type (#{ast_item.class})")
+            end
+          self[:children] << child if child
+        end
+      end
+    end
+
+    class ComponentPS < self
+      def initialize(ast_item, opts = {})
+        ast_item_type = ast_item.type
+        type =
+          case ast_item_type
+          when :hostclass
+            'class'
+          when :definition
+            'definition'
+          else
+            fail DTK::Client::Error.new('unexpected type for ast_item')
+          end
+
+        self['type'] = type
+        self['name'] = ast_item.name
+        attributes  = []
+
+        attributes << AttributePS.create_name_attribute if ast_item_type == :definition
+        (ast_item.arguments || []).each { |arg| attributes << AttributePS.create(arg, opts) }
+
+        self['attributes'] = attributes unless attributes.empty?
+        super
+      end
+    end
+
+    AttrTypes = {
+      Puppet::Pops::Model::LiteralHash => 'hash',
+      Puppet::Pops::Model::VariableExpression => 'string',
+      Puppet::Pops::Model::LiteralUndef => 'string',
+      Puppet::Pops::Model::LiteralBoolean => 'boolean',
+      Puppet::Pops::Model::LiteralString => 'string',
+      Puppet::Pops::Model::LiteralInteger => 'integer',
+      Puppet::Pops::Model::LiteralList => 'array',
+      Puppet::Pops::Model::ConcatenatedString => 'string',
+      Puppet::Pops::Model::QualifiedName => 'string'
+    }
+    class AttributePS < self
+      def initialize(arg, opts = {})
+        self['name'] = arg[0]
+
+        if arg_1 = arg[1]          
+          self['type']     = type?(arg_1)
+          self['required'] = opts[:required] if opts.key?(:required)
+
+          unless def_value(arg_1)
+            self['required'] ||= true
+          end
+        else
+          self['type']     = 'string'
+          self['required'] = true
+        end
+
+        super
+      end
+
+      def type?(arg)
+        if arg_value = arg.value
+          AttrTypes[arg_value.class] || 'string'
+        else
+          'string'
+        end
+      end
+
+      def def_value(default_ast_obj)
+        # if arg_value = default_ast_obj.value
+        #   default_ast_obj.source_text unless IgnoreValues.include?(arg_value.class)
+        # else
+        default_ast_obj.source_text
+        # end
+      end
+      # IgnoreValues = [Puppet::Pops::Model::VariableExpression]
+
+      def self.create_name_attribute
+        new(['name'], 'required' => true)
+      end
+    end
+
+  end
+end

data/lib/dtk_puppet/parser.rb

@@ -0,0 +1,92 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK
+  module Puppet
+    class Parser
+      def self.parse(argv)
+        parse_puppet_module_directory
+      end
+
+      def self.parse_puppet_module_directory
+        current_dir = Client::OsUtil.current_dir
+        dir_content = Dir.glob("#{current_dir}/**/*")
+
+        if module_yaml_file = dir_content.find { |path| path =~ /dtk.module.yaml$/ }
+          fail DTK::Client::Error::Usage.new("Dtk module file 'dtk.module.yaml' exists already. Please delete it first and execute puppet scaffolding again.")
+        end
+
+        metadata_file = dir_content.find { |path| path =~ /metadata.json$/ }
+        file_content  = File.read(metadata_file)
+        metadata_hash = JSON.parse(file_content)
+
+        full_module_name = metadata_hash['name'].gsub('-', '/')
+        module_version   = metadata_hash['version']
+        dependencies     = process_metadata_dependencies(metadata_hash['dependencies'])
+
+        module_hash = {
+          'module' => full_module_name,
+          'version' => module_version
+        }
+        module_hash.merge!('dependencies' => dependencies) unless dependencies.empty?
+
+        all_files = dir_content.select { |path| path =~ /manifests.+\.pp$/ }
+        if all_files.empty?
+          all_files = dir_content.select { |path| path =~ /puppet\/manifests.+\.pp$/ }
+        end
+
+        namespace, module_name = full_module_name.split('/')
+
+        component_defs = process_manifest_files(all_files, module_name)
+        module_hash['component_defs'] = component_defs
+
+        File.open("#{current_dir}/dtk.module.yaml", 'wb') { |file| file.write(module_hash.to_yaml) }
+        DTK::Client::OsUtil.print_info("'dtk.module.yaml' file has been created successfully.")
+      end
+
+      def self.process_manifest_files(files, module_name)
+        ret    = ParseStructure::TopPS.new()
+        parser = ::Puppet::Parser::ParserFactory.parser
+
+        files.each do |file|
+          parser.file    = file
+          initial_import = parser.parse
+
+          known_resource_types = ::Puppet::Resource::TypeCollection.new('production')
+          known_resource_types.import_ast(initial_import, '')
+          krt_code = known_resource_types.hostclass('').code
+
+          ret.add_children(krt_code)
+        end
+
+        ret.render_hash_form(module_name)
+      end
+
+      def self.process_metadata_dependencies(dependencies)
+        dependency_hash = {}
+
+        if dependencies
+          dependencies.each do |dependency|
+            dependency_hash.merge!(dependency['name'] => 'master')
+          end
+        end
+
+        dependency_hash
+      end
+    end
+  end
+end

data/lib/dtk_puppet.rb

@@ -0,0 +1,24 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK
+  module Puppet
+    require_relative('dtk_puppet/parser')
+    require_relative('dtk_puppet/parse_structure')
+    require_relative('client/error')
+  end
+end