dtk-client 0.9.1 → 0.10.0

data/lib/client/config.rb

@@ -0,0 +1,106 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'singleton'
+
+module DTK::Client
+  ##
+  # Singleton patern to hold configuration for dtk client.
+  #
+  class Config < Hash
+    include Singleton
+    
+    def self.[](k)
+      instance[k.to_s]
+    end
+
+    CLIENT_CONF            = 'client.conf'
+    DEFAULT_CONF_FILE_PATH = File.expand_path('config/default.conf', File.dirname(__FILE__))
+
+    private
+
+    def initialize
+      set_defaults!
+      load_config_file_values!
+      validate
+    end
+
+    def set_defaults!
+      merge_config_file_content_into_hash!(DEFAULT_CONF_FILE_PATH)
+    end
+    
+    def load_config_file_values!
+      client_config_path = Configurator.client_config_path
+      merge_config_file_content_into_hash!(client_config_path) if File.exist?(client_config_path)
+    end
+
+    REQUIRED_KEYS = ['server_host']
+    def validate
+      # TODO: along with checking for missing keys should check for legal values
+      missing_keys = REQUIRED_KEYS - keys
+      unless missing_keys.empty?
+        raise Error::Usage, "Missing config keys (#{missing_keys.join(", ")}) in client config file '#{Configurator.client_config_path}"
+      end
+    end
+
+    def merge_config_file_content_into_hash!(path)
+      merge!(parse_string_content(File.read(path)))
+    end
+
+    # returns a hash
+    def parse_string_content(string)
+      ret = {}
+      string.each_line do |line|
+        line.strip!
+        # strip off comment
+        line.gsub!(/#.+$/,'')
+        # remove control characters 
+        line.gsub!(/\t/,'')
+        # below strips blanks after and before '='
+        if line =~ /(^[^=]+)[ ]*=[ ]*(.+$)/
+          attr = $1
+          val_string = $2
+          ret.merge!(attr => parse_value_string(val_string))
+        else
+          # skipping any line that does not parse
+        end
+      end
+      ret
+    end
+
+    def parse_value_string(val_string)
+      # remove trailing blanks
+      val_string = val_string.gsub(/[ ]+$/,'')
+      convert_data_types(val_string)
+    end
+
+    def convert_data_types(val_string)
+      case val_string
+       when /^(true|false)$/ 
+        val_string.eql?('true') ? true : false
+       when /^[0-9]+$/
+        val_string.to_i
+       when /^[0-9\.]+$/
+        # making sure dont do something like '10.0.0.1'.to_f
+        val_string.split('.').size == 2 ? val_string.to_f : val_string
+       else 
+        val_string
+      end
+    end
+  end
+end
+

data/lib/client/configurator.rb

@@ -0,0 +1,182 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'fileutils'
+module DTK::Client
+  module Configurator
+    CONFIG_FILE   = File.join(DtkPath.base_dir, 'client.conf')
+    CRED_FILE     = File.join(DtkPath.base_dir, '.connection')
+    DIRECT_ACCESS = File.join(DtkPath.base_dir, '.add_direct_access')
+    NODE_SSH_CREDENTIALS = File.join(DtkPath.base_dir, 'ssh_credentials.yaml')
+
+    def self.client_config_path
+      CONFIG_FILE
+    end
+
+    def self.get_credentials
+      cred_file = CRED_FILE
+      raise Error, "Authorization configuration file (#{cred_file}) does not exist" unless File.exists?(cred_file)
+      ret = parse_key_value_file(cred_file)
+      [:username, :password].each{ |k| raise Error, "cannot find #{k}" unless ret[k] }
+      ret
+    end
+
+    def self.create_missing_client_dirs
+      base_dir = DtkPath.base_dir
+      FileUtils.mkdir(base_dir) unless File.directory?(base_dir)
+    end
+
+    def self.check_config_exists
+      exists = true
+      if !File.exists?(client_config_path)
+        puts "", "Please enter the DTK server address (example: instance.dtk.io)"
+        header = File.read(Config::DEFAULT_CONF_FILE_PATH)
+        generate_conf_file(client_config_path, [['server_host', 'Server address']], header)
+        exists = false
+      end
+      if !File.exists?(CRED_FILE)
+        puts "", "Please enter your DTK login details"
+        generate_conf_file(CRED_FILE, [['username', 'Username'], ['password', 'Password']], '')
+        exists = false
+      end
+      
+      exists
+    end
+    
+    def self.check_git
+      if OsUtil.which('git') == nil
+        OsUtil.put_warning "[WARNING]", "Can't find the 'git' command in you path. Please make sure git is installed in order to use all features of DTK Client.", :yellow
+      else
+        OsUtil.put_warning "[WARNING]", 'Git username not set. This can cause issues while using DTK Client. To set it, run `git config --global user.name "User Name"`', :yellow if `git config --get user.name` == ""
+        OsUtil.put_warning "[WARNING]", 'Git email not set. This can cause issues while using DTK Client. To set it, run `git config --global user.email "me@here.com"`', :yellow if `git config --get user.email` == ""
+      end
+    end
+    
+    # return true/false, .add_direct_access file location and ssk key file location
+    def self.check_direct_access
+      username_exists  = check_for_username_entry(client_username)
+      ssh_key_path = SSHUtil.default_rsa_pub_key_path
+      
+      {:username_exists => username_exists, :file_path => DIRECT_ACCESS, :ssh_key_path => ssh_key_path}
+    end
+
+    def self.regenerate_conf_file(file_path, properties, header)
+      File.open(file_path, 'w') do |f|
+        f.puts(header)
+        properties.each do |prop|
+          f.puts("#{prop[0]}=#{prop[1]}")
+        end
+      end
+    end
+    
+    def self.parse_key_value_file(file)
+      # adapted from mcollective config
+      ret = Hash.new
+      raise Error,"Config file (#{file}) does not exists" unless File.exists?(file)
+      File.open(file).each do |line|
+        # strip blank spaces, tabs etc off the end of all lines
+        line.gsub!(/\s*$/, "")
+        unless line =~ /^#|^$/
+          if (line =~ /(.+?)\s*=\s*(.+)/)
+            key = $1
+            val = $2
+            ret[key.to_sym] = val
+          end
+        end
+      end
+      ret
+    end
+
+    def self.add_current_user_to_direct_access
+      username = client_username
+      
+      File.open(DIRECT_ACCESS, 'a') do |file|
+        file.puts(username)
+      end
+      
+      true
+    end
+    
+    def self.client_username
+      parse_key_value_file(CRED_FILE)[:username]
+    end
+
+    #
+    # Method will check if there is username entry in DIRECT_ACCESS file
+    #
+    def self.check_for_username_entry(username)
+      if File.exists?(DIRECT_ACCESS)
+        File.open(DIRECT_ACCESS).each do |line|
+          if line.strip.eql?(username)
+            return true
+            end
+        end
+      end
+      
+      false
+    end
+    
+    def self.ask_catalog_credentials
+      are_there_creds = Console.prompt_yes_no("Do you have DTK catalog credentials?", :add_options => true)
+      property_template = {}
+      if are_there_creds
+        property_template = self.enter_catalog_credentials
+      end
+      
+      property_template
+    end
+    
+    def self.enter_catalog_credentials
+      property_template = {}
+      # needed to preserve the order for ruby 1.8.7
+      # ruby 1.8 does not preserve order of insertation
+      wizard_values = { :username => 'Catalog Username', :password => 'Catalog Password' }
+      [:username, :password].each do |p|
+        value = ask("#{wizard_values[p]}: ") { |q| q.echo = false if p == :password }
+        property_template.store(p, value)
+      end
+      property_template
+    end
+
+    private
+
+    def self.generate_conf_file(file_path, properties, header)
+      require 'highline/import'
+      property_template = []
+      
+      properties.each do |p,d|
+        begin
+          trap("INT") {
+            puts "", "Exiting..."
+            abort
+          }
+        end
+        value = ask("#{d}: ") { |q| q.echo = false if p == 'password'}
+        property_template << [p,value]
+      end
+
+      File.open(file_path, 'w') do |f|
+        f.puts(header)
+        property_template.each do |prop|
+          f.puts("#{prop[0]}=#{prop[1]}")
+        end
+      end
+    end
+
+
+  end
+end

data/lib/client/conn.rb

@@ -0,0 +1,197 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module DTK::Client
+  class Conn
+    def initialize
+      @cookies          = {}
+      @connection_error = nil
+      login
+    end
+    
+    attr_reader :connection_error, :cookies
+    
+    def self.get_timeout
+      default_rest_opts[:timeout]
+    end
+    
+    def self.set_timeout(timeout_sec)
+      default_rest_opts[:timeout] = timeout_sec
+    end
+    
+    def get_username
+      get_credentials[:username]
+    end
+    
+    def get(route, query_string_hash = {})
+      url = rest_url(route)
+      ap "GET #{url}" if verbose_mode_on?
+      
+      check_and_wrap_response { json_parse_if_needed(get_raw(url, query_string_hash)) }
+    end
+    
+    def post(route, post_body = {})
+      url = rest_url(route)
+      if verbose_mode_on?
+        ap "POST (REST) #{url}"
+        ap "params: "
+        ap post_body
+      end
+      
+      check_and_wrap_response { json_parse_if_needed(post_raw(url, post_body)) }
+    end
+    
+    def post_file(route, post_body = {})
+      url = rest_url(route)
+      if verbose_mode_on?
+          ap "POST (FILE) #{url}"
+        ap "params: "
+        ap post_body
+      end
+      
+      check_and_wrap_response { json_parse_if_needed(post_raw(url,post_body,{:content_type => 'avro/binary'})) }
+    end
+
+    def connection_error?
+      !@connection_error.nil?
+    end
+    
+    ##
+    # Method will warn user that connection could not be established. User should check configuration
+    # to make sure that connection is properly set.
+    #
+    def print_warning
+      creds = get_credentials
+      puts   "[ERROR] Unable to connect to server, please check you configuration."
+      puts   "========================== Configuration =========================="
+      printf "%15s %s\n", "REST endpoint:", rest_url
+      printf "%15s %s\n", "Username:", "#{creds[:username]}"
+      printf "%15s %s\n", "Password:", "#{creds[:password] ? creds[:password].gsub(/./,'*') : 'No password set'}"
+      puts   "==================================================================="
+      
+      if connection_error['errors'].first['errors']
+        error_code = self.connection_error['errors'].first['errors'].first['code']
+        OsUtil.print_error("Error code: #{error_code}")
+      end
+    end
+    
+    private
+
+    REST_VERSION = 'v1'
+    REST_PREFIX = "rest/api/#{REST_VERSION}"
+    # REST_PREFIX = "rest"
+
+    def rest_url(route = nil)
+      "#{rest_url_base}/#{REST_PREFIX}/#{route}"
+    end
+
+    def rest_url_base
+      @@rest_url_base ||= get_rest_url_base
+    end
+
+    def get_rest_url_base
+      protocol, port = 
+        if "#{Config[:secure_connection]}" == 'true'
+          ['https', Config[:secure_connection_server_port].to_s]
+        else
+          ['http', Config[:server_port].to_s]
+        end
+      "#{protocol}://#{Config[:server_host]}:#{port}"
+    end
+    
+    # method will repeat request in case session has expired
+    def check_and_wrap_response(&rest_method_func)
+      response = rest_method_func.call
+      
+      if Response::ErrorHandler.check_for_session_expiried(response)
+        # re-logging user and repeating request
+        OsUtil.print_warning("Session expired: re-establishing session & re-trying request ...")
+        @cookies = Session.re_initialize
+        response = rest_method_func.call
+      end
+      
+      response_obj = Response.new(response)
+      
+      # queue messages from server to be displayed later
+      #TODO: DTK-2554: put in processing of messages Shell::MessageQueue.process_response(response_obj)
+      response_obj
+    end
+
+    def verbose_mode_on?
+      if @verbose_mode_on.nil?
+        if @verbose_mode_on ||= !!Config[:verbose_rest_calls]
+          require 'ap'
+        end
+      end
+      @verbose_mode_on
+    end
+
+    def login
+      response = post_raw rest_url('auth/login'), get_credentials
+      if response.kind_of?(::DTK::Common::Response) and ! response.ok?
+        @connection_error = response
+      else
+        @cookies = response.cookies
+      end
+    end
+    
+    def logout
+      response = get_raw rest_url('auth/logout')
+      # TODO: see if response can be nil
+      raise Error, "Failed to logout, and terminate session!" unless response
+      @cookies = nil
+    end
+
+    def set_credentials(username, password)
+      @parsed_credentials = { :username => username, :password => password}
+    end
+    
+    def get_credentials
+      @parsed_credentials ||= Configurator.get_credentials
+    end
+
+    def default_rest_opts
+      @default_rest_opts ||= get_default_rest_opts
+    end
+
+    def get_default_rest_opts
+      # In development mode we want bigger timeout allowing us to debbug on server while still
+      # keeping connection alive and receivinga response
+      timeout = Config[:development_mode] ? 2000 : 150
+      {
+        :timeout => timeout,
+        :open_timeout => 10,
+        :verify_ssl => OpenSSL::SSL::VERIFY_PEER,
+        :ssl_ca_file => File.expand_path('../client/config/cacert.pem', File.dirname(__FILE__)),
+      }
+    end
+    
+    def get_raw(url, query_string_hash = {})
+      Response::RestClientWrapper.get_raw(url, query_string_hash, default_rest_opts.merge(:cookies => @cookies))
+    end
+
+    def post_raw(url, post_body, params = {})
+      Response::RestClientWrapper.post_raw(url, post_body, default_rest_opts.merge(:cookies => @cookies).merge(params))
+    end
+    
+    def json_parse_if_needed(item)
+      Response::RestClientWrapper.json_parse_if_needed(item)
+    end
+  end
+end
+

data/lib/client/content_generator.rb

@@ -0,0 +1,127 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'yaml'
+
+module DTK::Client
+  # Abstract class that holds classes and methods for xecuting commands by
+  # make calls to server and performing client side operations
+  class ContentGenerator
+    def initialize(content_dir, module_ref, version)
+      @content_dir       = content_dir
+      @module_ref        = module_ref
+      @version           = version
+      @directory_content = nil
+    end
+
+    def generate_module_content
+      module_hash = {
+        'dsl_version' => '1.0.0',
+        'module'  => "#{@module_ref.namespace}/#{@module_ref.module_name}",
+        'version' => @version || 'master'
+      }
+
+      if dependencies = ret_dependencies_hash
+        module_hash.merge!('dependencies' => dependencies)
+      end
+
+      if assemblies = ret_assemblies_hash
+        module_hash.merge!('assemblies' => assemblies)
+      end
+
+      module_hash
+    end
+
+    private
+
+    def get_raw_content?(file_path)
+      File.open(file_path).read if file_path and File.exists?(file_path)
+    end
+
+    def convert_file_content_to_hash(file_path)
+      begin
+        YAML.load(get_raw_content?(file_path))
+      rescue Exception => e
+        yaml_err_msg = e.message.gsub(/\(<unknown>\): /,'').capitalize 
+        raise Error::Usage, "YAML parsing error in '#{file_path}':\n#{yaml_err_msg}"
+      end
+    end
+
+    def get_directory_content
+      @directory_content ||= Dir.glob("#{@content_dir}/**/*")
+    end
+
+    def invalidate_directory_content
+      @directory_content = Dir.glob("#{@content_dir}/**/*")
+    end
+
+    def get_assembly_files
+      get_directory_content.select { |f| f =~ AssemblyRegexp[:regexp] || f =~ AssemblyRegexp[:legacy_regexp] }
+    end
+    AssemblyRegexp = {
+      :regexp        => Regexp.new("assemblies/(.*)\.dtk\.assembly\.(yml|yaml)$"),
+      :legacy_regexp => Regexp.new("assemblies/([^/]+)/assembly\.(yml|yaml)$")
+    }
+
+    def get_module_refs_file
+      get_directory_content.find { |f| f =~ ModuleRefsRegexp[:regexp] }
+    end
+    ModuleRefsRegexp = { 
+      :regexp => Regexp.new("module_refs\.(yml|yaml)$")
+    }
+
+    def ret_assemblies_hash
+      assemblies = {}
+
+      get_assembly_files.each do |assembly|
+        content_hash     = convert_file_content_to_hash(assembly)
+        name             = content_hash['name']
+        assembly_content = content_hash['assembly']
+        workflows        = ret_workflows_hash(content_hash['workflow'])
+        
+        assembly_content.merge!('workflows' => workflows) if workflows
+        assemblies.merge!(name => assembly_content)
+      end
+
+      assemblies.empty? ? nil : assemblies
+    end
+
+    def ret_workflows_hash(workflow)
+      return if workflow.nil? || workflow.empty?
+      workflow_name = workflow.delete('assembly_action')
+      {
+        workflow_name => workflow
+      }
+    end
+
+    def ret_dependencies_hash
+      if file_path = get_module_refs_file
+        module_refs_content = convert_file_content_to_hash(file_path)
+        dependencies = []
+
+        if cmp_dependencies = module_refs_content['component_modules']
+          cmp_dependencies.each_pair do |name, namespace_h|
+            dependencies << "#{namespace_h['namespace']}/#{name}"
+          end
+        end
+
+        dependencies
+      end
+    end
+  end
+end
+