dtk-shell 0.10.0

data/lib/util/permission_util.rb

@@ -0,0 +1,31 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK
+  module Client
+    module PermissionUtil
+      class << self
+        def validate_permissions!(permission_string)
+          # matches example: u-rw, ugo+r, go+w
+          match = permission_string.match(/^[ugo]+[+\-][rwd]+$/)
+          raise DTK::Client::DtkValidationError, "Provided permission expression ('#{permission_string}') is not valid" unless match
+          permission_string
+        end
+      end
+    end
+  end
+end

data/lib/util/remote_dependency_util.rb

@@ -0,0 +1,84 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# Managment of remote dependencies detection (print warning) or providing data from local resources
+#
+module DTK
+  module Client
+    module RemoteDependencyUtil
+      MODULE_REF_FILE = 'module_refs.yaml'
+
+      class << self
+        def print_dependency_warnings(response, success_msg = nil, opts = {})
+          are_there_warnings = false
+          return are_there_warnings if response.nil? || response.data.nil?
+
+          warnings = response.data['dependency_warnings']
+          if warnings && !warnings.empty?
+            if opts[:ignore_permission_warnings]
+              warnings.delete_if { |warning| warning['error_type'].eql?('no_permission') }
+              return if warnings.empty?
+            end
+            print_out "Following warnings have been detected for current module by Repo Manager:\n"
+            warnings.each { |w| print_out("  - #{w['message']}") }
+            puts
+            are_there_warnings = true
+          end
+          print_out success_msg, :green if success_msg
+          are_there_warnings
+        end
+
+        def check_permission_warnings(response)
+          errors = ''
+          dependency_warnings = response.data['dependency_warnings']
+
+          if dependency_warnings && !dependency_warnings.empty?
+            no_permissions = dependency_warnings.select { |warning| warning['error_type'].eql?('no_permission') }
+
+            unless no_permissions.empty?
+              errors << "\n\nYou do not have (R) permissions for modules:\n\n"
+              no_permissions.each { |np| errors << "  - #{np['module_namespace']}:#{np['module_name']} (owner: #{np['module_owner']})\n" }
+              errors << "\nPlease contact owner(s) to change permissions for those modules."
+            end
+          end
+
+          raise DtkError, errors unless errors.empty?
+        end
+
+        # check if all dependent modules are frozen; if they are don't display prompt for update
+        def check_for_frozen_modules(required_modules)
+          return true if required_modules.nil? || required_modules.empty?
+
+          modules_to_update = required_modules.select{ |md| (md['frozen'].nil? || md['frozen'] == false)}
+          return modules_to_update.empty?
+        end
+
+        def module_ref_content(location)
+          abs_location = File.join(location, MODULE_REF_FILE)
+          File.exists?(abs_location) ? File.read(abs_location) : nil
+        end
+
+        private
+
+        def print_out(message, color=:yellow)
+          DTK::Client::OsUtil.print(message, color)
+        end
+      end
+    end
+  end
+end

data/lib/util/ssh_util.rb

@@ -0,0 +1,94 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK
+  module Client
+    module SSHUtil
+
+      def self.read_and_validate_pub_key(path_to_pub_key)
+        is_ssh_key_path_valid?(path_to_pub_key)
+        rsa_pub_key = File.open(path_to_pub_key) { |f| f.read }
+        is_ssh_key_content_valid?(rsa_pub_key)
+        rsa_pub_key
+      end
+
+      def self.update_ssh_known_hosts(server_dns,server_fingerprint)
+        known_hosts_path = ssh_known_hosts_path()
+        if File.file?(known_hosts_path)
+          `ssh-keygen -f #{known_hosts_path} -R #{server_dns} 2> /dev/null`
+          File.open(known_hosts_path,"a"){|f|f << server_fingerprint}
+        else
+          ssh_base_dir = ssh_base_dir()
+          unless File.directory?(ssh_base_dir)
+            Dir.mkdir(ssh_base_dir)
+          end
+          File.open(known_hosts_path,"w"){|f|f << server_fingerprint}
+        end
+      end
+      
+      def self.default_rsa_pub_key_path()
+        "#{ssh_base_dir()}/id_rsa.pub"
+      end
+
+      def self.rsa_pub_key_content()
+        path_to_key = self.default_rsa_pub_key_path()
+        unless File.file?(path_to_key)
+          raise DtkError,"No File found at (#{path_to_key}). Path is wrong or it is necessary to generate the public rsa key (e.g., run ssh-keygen -t rsa)"
+        end
+
+        content = File.open(path_to_key){ |f| f.read }
+        content.chomp
+      end
+
+      def self.ssh_reachable?(user, server, timeout = 3)
+        output = `ssh -o ConnectTimeout=#{timeout} -o \"StrictHostKeyChecking no\" -o \"UserKnownHostsFile /dev/null\" -q #{user}@#{server} exit; echo $?`
+
+        # if response 0 than it is able to connect
+        "0".eql?(output.strip())
+      end
+
+
+    private
+
+      def self.ssh_base_dir()
+        "#{OsUtil.dtk_home_dir}/.ssh"
+      end
+      
+      def self.ssh_known_hosts_path()
+        "#{ssh_base_dir()}/known_hosts"
+      end
+
+      def self.is_ssh_key_path_valid?(path_to_key)
+        unless path_to_key.include?(".pub")
+          raise DtkError, "[ERROR] Invalid public key file path (#{path_to_key}). Please provide valid path and try again."
+        end
+
+        unless File.exists?(path_to_key)
+          raise DtkError, "[ERROR] Not able to find provided key (#{path_to_key}). Please provide valid path and try again."
+        end
+      end
+
+      def self.is_ssh_key_content_valid?(rsa_pub_key)
+        # checking know ssh rsa pub key content
+        if(rsa_pub_key.empty? || !rsa_pub_key.include?("AAAAB3NzaC1yc2EA"))
+          raise DtkError, "[ERROR] SSH public key (#{path_to_key}) does not have valid content. Please check your key and try again."
+        end
+      end
+
+    end
+  end
+end

data/lib/view_processor.rb

@@ -0,0 +1,129 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK
+  module Client
+    module RenderView
+      SIMPLE_LIST     = "simple_list"
+      TABLE           = "table_print"
+      PRETTY_PRINT    = "hash_pretty_print"
+      AUG_SIMPLE_LIST = "augmented_simple_list"
+    end
+
+    class ViewProcessor
+      class << self
+        include Auxiliary
+        def render(command_class, ruby_obj, type, data_type, adapter=nil, print_error_table=false)
+          adapter ||= get_adapter(type,command_class,data_type)
+          if type == RenderView::TABLE
+            # for table there is only one rendering, we use command class to
+            # determine output of the table
+            adapter.render(ruby_obj, command_class, data_type, nil, print_error_table)
+              
+            # saying no additional print needed (see core class)
+            return false
+          elsif ruby_obj.kind_of?(Hash)
+            adapter.render(ruby_obj)
+          elsif ruby_obj.kind_of?(Array)
+            ruby_obj.map{|el|render(command_class,el,type,nil,adapter)}
+          elsif ruby_obj.kind_of?(String)
+            ruby_obj
+          else
+            raise Error.new("ruby_obj has unexepected type")
+          end
+        end
+
+        def get_adapter(type,command_class,data_type=nil)
+
+          data_type_index = use_data_type_index?(command_class,data_type)
+          cached = 
+            if data_type_index
+              ((AdapterCacheAug[type]||{})[command_class]||{})[data_type_index]
+            else
+              (AdapterCache[type]||{})[command_class]
+            end               
+
+          return cached if cached
+          dtk_nested_require("view_processor",type)
+          klass = DTK::Client.const_get "ViewProc#{cap_form(type)}" 
+          if data_type_index
+            AdapterCacheAug[type] ||= Hash.new
+            AdapterCacheAug[type][command_class] ||= Hash.new
+            AdapterCacheAug[type][command_class][data_type_index] = klass.new(type,command_class,data_type_index)
+          else
+            AdapterCache[type] ||= Hash.new
+            AdapterCache[type][command_class] = klass.new(type,command_class)               
+          end
+        end
+            
+        AdapterCache = Hash.new
+        AdapterCacheAug = Hash.new
+      end
+     private
+      def initialize(type,command_class,data_type_index=nil)
+        @command_class = command_class
+        @data_type_index = data_type_index
+      end
+
+      #data_type_index is used if there is adata type passed and it is different than command_class defualt data type
+      def self.use_data_type_index?(command_class,data_type)
+        if data_type
+         data_type_index = data_type.downcase
+          if data_type_index != snake_form(command_class)
+            data_type_index
+          end
+        end
+      end
+
+      def get_meta(type,command_class,data_type_index=nil)
+        ret = nil
+        view = data_type_index||snake_form(command_class)
+        view = command_class if view.empty?
+        # TODO: Fix this logic, but we first need to see what to do with simple lists
+        if type.eql?('hash_pretty_print')
+          return pretty_print_meta(command_class, data_type_index)
+        end
+   
+        begin
+          dtk_require("../views/#{view}/#{type}")
+          view_const = DTK::Client::ViewMeta.const_get cap_form(view)
+          ret = view_const.const_get cap_form(type)
+        rescue Exception
+          ret = failback_meta(command_class.respond_to?(:pretty_print_cols) ? command_class.pretty_print_cols() : [])
+        end
+           
+        return ret
+      end
+
+      def pretty_print_meta(command_class,data_type_index=nil)
+        view = data_type_index||snake_form(command_class)
+        view = command_class if view.empty?
+        # content = DiskCacher.new.fetch("http://localhost/mockup/get_pp_metadata", ::DTK::Configuration.get(:meta_table_ttl))
+        content = DiskCacher.new.fetch("pp_metadata", ::DTK::Configuration.get(:meta_table_ttl))
+        raise DTK::Client::DtkError, "Pretty print metadata is empty, please contact DTK team." if content.empty?
+        hash_content = JSON.parse(content, {:symbolize_names => true})
+        (view && (not view.empty?) && hash_content[view.to_sym])||empty_pretty_print_meta()
+      end
+
+      def empty_pretty_print_meta()
+        {:top_type=>:top, :defs=>{:top_def=>[]}}
+      end
+    end
+    module ViewMeta
+    end
+  end
+end

data/lib/view_processor/augmented_simple_list.rb

@@ -0,0 +1,44 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#TODO: test for assembly list/display; want to make assembly specfic stuff datadriven
+dtk_require 'simple_list'
+module DTK
+  module Client
+    class ViewProcAugmentedSimpleList < ViewProcSimpleList
+     private
+      def initialize(type,command_class,data_type_index=nil)
+        super
+        @meta = get_meta(type,command_class)
+      end
+      def failback_meta(ordered_cols)
+        nil
+      end
+      def simple_value_render(ordered_hash,ident_info)
+        augmented_def?(ordered_hash,ident_info) || super
+      end
+      def augmented_def?(ordered_hash,ident_info)
+        return nil unless @meta
+        if aug_def =  @meta["#{ordered_hash.object_type}_def".to_sym]
+          ident_str = ident_str(ident_info[:ident]||0)
+          vals = aug_def[:keys].map{|k|ordered_hash[k.to_s]}
+          "#{ident_str}#{aug_def[:fn].call(*vals)}\n"
+        end
+      end
+    end
+  end
+end

data/lib/view_processor/hash_pretty_print.rb

@@ -0,0 +1,123 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK
+  module Client
+    class ViewProcHashPrettyPrint < ViewProcessor
+      include ::DTK::Client::Auxiliary
+      def render(hash)
+        object_def = get_top_def()
+        raise_error() unless object_def
+        render_object_def(hash,object_def)
+      end
+     private
+      attr_reader :meta
+      def initialize(type,command_class,data_type_index=nil)
+        super
+        @meta = get_meta(type,command_class,data_type_index)
+      end
+
+      def failback_meta(ordered_cols)
+        {
+          :top_type => :top,
+          :defs => {:top_def => ordered_cols}
+        }
+      end
+
+      def get_top_def()
+        raise_error("No Top def") unless top_object_type = meta[:top_type]
+        get_object_def(top_object_type)
+      end
+
+      def get_object_def(object_type)
+        if defs = meta[:defs] and object_type
+          {object_type => defs["#{object_type}_def".to_sym]}
+        end
+      end
+      def raise_error(msg=nil)
+        msg ||= "No hash pretty print view defined"
+        raise Error.new(msg)
+      end
+      def render_object_def(object,object_def,opts={})
+        #TODO: stub making it only first level
+        return object unless object.kind_of?(Hash)
+        hash = object
+        ret = ViewPrettyPrintHash.new(object_def.keys.first)
+
+        object_def.values.first.each do |item|
+          if item.kind_of?(Hash)
+            render_object_def__hash_def!(ret,hash,item)    
+          else
+            key = item.to_s
+            target_key = replace_with_key_alias?(key) 
+            #TODO: may want to conditionally include nil values
+            ret[target_key] = hash[key] if hash[key]
+          end
+        end
+        #catch all for keys not defined
+        unless opts[:only_explicit_cols]
+          (hash.keys.map{|k|replace_with_key_alias?(k)} - ret.keys).each do |key|
+            ret[key] = hash[key] if hash[key]
+          end
+        end
+        return ret
+      end
+      def replace_with_key_alias?(key)
+        #TODO: fix
+        return key
+        if ret = GlobalKeyAliases[key.to_sym] then ret.to_s 
+        else key
+        end
+      end
+      GlobalKeyAliases = {
+        :library_library_id => :library_id,
+        :datacenter_datacenter_id => :target_id
+      }
+
+      def render_object_def__hash_def!(ret,hash,hash_def_item)
+        key = hash_def_item.keys.first.to_s
+        return unless input = hash[key]
+        hash_def_info = hash_def_item.values.first
+        nested_object_def = get_object_def(hash_def_info[:type])
+        raise_error("object def of type (#{hash_def_info[:type]||""}) does not exist") unless nested_object_def
+
+        opts = Hash.new
+        if hash_def_info[:only_explicit_cols]
+          opts.merge!(:only_explicit_cols => true)
+        end
+        if hash_def_info[:is_array]
+          raise_error("hash subpart should be an array") unless input.kind_of?(Array)
+          ret[key] = input.map{|el|render_object_def(el,nested_object_def,opts)}
+        else
+          ret[key] = render_object_def(input,nested_object_def,opts)
+        end
+      end
+    end
+    class ViewPrettyPrintHash < Common::PrettyPrintHash
+      def initialize(object_type=nil)
+        super()
+        @object_type = object_type
+      end
+      attr_accessor :object_type
+      def slice(*keys)
+        ret = super
+        ret.object_type = object_type
+        ret
+      end
+    end
+  end
+end