dtk-shell 0.10.0

data/lib/config/client.conf.header

@@ -0,0 +1,20 @@
+development_mode=false
+meta_table_ttl=7200000            # time to live (ms)
+meta_constants_ttl=7200000        # time to live (ms)
+meta_pretty_print_ttl=7200000     # time to live (ms)
+task_check_frequency=60           # check frequency for task status threads (seconds)
+tail_log_frequency=2              # assembly - frequency between requests (seconds)
+debug_task_frequency=5            # assembly - frequency between requests (seconds)
+auto_commit_changes=false         # autocommit for modules
+verbose_rest_calls=false          # logging of REST calls
+
+# if relative path is used we will use HOME + relative path, apsoluth path will override this
+module_location=component_modules
+service_location=service_modules
+test_module_location=test_modules
+backups_location=backups
+
+# server connection details
+server_port=80
+secure_connection_server_port=443
+secure_connection=true

data/lib/config/configuration.rb

@@ -0,0 +1,99 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# Singleton patern to hold configuration for dtk client.
+#
+# DEFAULT:     Configuration 
+# DEVELOPMENT: Can add lib/config/local.conf, this is git ignored
+#
+# NOTE: Singleton here is not necessery since it will not persist in memory due
+#       to nature of DTK Client, but it is right approach for possible re-use
+#
+# Pririoty of load
+#   1) LOCAL
+#   2) EXTERNAL
+#   3) DEFAULT
+#
+# NOTE: Default will be used if there some parameters missing in other configuration
+#
+require 'singleton'
+
+dtk_require_from_base('util/os_util')
+
+module DTK
+  class Configuration
+    include Singleton
+  
+    EXTERNAL_APP_CONF = "client.conf"
+    DEVELOPMENT_CONF  = 'local.conf'
+    DEFAULT_CONF      = 'default.conf'
+
+    def self.get(name, default=nil)
+      Configuration.instance.get(name, default)
+    end
+
+    def self.[](k)
+      Configuration.instance.get(k)
+    end
+    
+    def initialize
+      # default configuration
+      @cache = load_configuration_to_hash(File.expand_path("../#{DEFAULT_CONF}",__FILE__))
+
+      # we will not use local.conf from gemfile because client.conf is required so this is deprecated
+      if File.exist?(File.expand_path("../#{DEVELOPMENT_CONF}",__FILE__))
+        local_configuration = load_configuration_to_hash(File.expand_path("../#{DEVELOPMENT_CONF}",__FILE__))
+        # we override only values from local configuration
+        # that way developer does not have updates its local configuration all the time
+        @cache.merge!(local_configuration)
+        # if we have loaded local configuration we will not check external
+        return
+      end
+
+      # We load this if there is no local configuration
+      external_file_location = File.join(::DTK::Client::OsUtil.dtk_local_folder(), "#{EXTERNAL_APP_CONF}")
+
+      if File.exist?(external_file_location)
+        external_configuration = load_configuration_to_hash(external_file_location)
+        @cache.merge!(external_configuration)
+      end
+    end
+
+    def get(name, default=nil)
+      return @cache[name.to_s] || default
+    end
+
+    private
+
+    def load_configuration_to_hash(path_to_file)
+      configuration = Hash[*File.read(path_to_file).gsub(/#.+/,'').strip().gsub(/( |\t)+$/,'').split(/[=\n\r\r\n]+/)]
+
+      # check for types
+      return configuration.each do |k,v|
+        case v
+          when /^(true|false)$/ 
+            configuration[k] = v.eql?('true') ? true : false
+          when /^[0-9]+$/
+            configuration[k] = v.to_i
+          when /^[0-9\.]+$/
+            configuration[k] = v.to_f
+        end
+      end
+    end
+  end
+end

data/lib/config/default.conf

@@ -0,0 +1,16 @@
+development_mode=false
+debug_grit=false
+meta_table_ttl=7200000     				# time to live (ms)
+meta_constants_ttl=7200000     		# time to live (ms)
+meta_pretty_print_ttl=7200000     # time to live (ms)
+task_check_frequency=60     			# check frequency for task status threads (seconds)
+tail_log_frequency=2      				# assembly - frequency between requests (seconds)
+debug_task_frequency=5     				# assembly - frequency between requests (seconds)
+auto_commit_changes=false         # autocommit for modules
+verbose_rest_calls=false          # logging of REST calls
+
+# if relative path is used we will use HOME + relative path, apsoluth path will override this
+module_location=component_modules
+service_location=service_modules
+test_module_location=test_modules
+backups_location=backups

data/lib/config/disk_cacher.rb

@@ -0,0 +1,80 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'net/http'
+#require 'md5' => Ruby 1.8.7 specific
+require File.expand_path('../util/os_util', File.dirname(__FILE__))
+dtk_require("../commands")
+
+
+#
+# Class dedicated for caching data on local system as well as for cookie management
+#
+class DiskCacher
+  include DTK::Client::CommandBase
+  extend DTK::Client::CommandBase
+
+  # file name to hold cookies
+  COOKIE_HOLDER_NAME = 'tempdtkstore'
+  FILE_DELIMITER = '--'
+
+  def initialize(cache_dir = DTK::Client::OsUtil.get_temp_location)
+    @cache_dir = cache_dir
+    @current_user = ::DTK::Client::Configurator.client_username
+  end
+
+  def fetch(file_name, max_age = 0, use_mock_up = true)
+    file = Digest::MD5.hexdigest(file_name)
+    # current user is important so that there are no clashes in writting to temp file
+    # between multiple users on same machine
+    file_path = File.join(@cache_dir, "#{@current_user}#{FILE_DELIMITER}#{file}")
+
+    # we check if the file -- a MD5 hexdigest of the URL -- exists
+    #  in the dir. If it does and the data is fresh, we just read
+    #  data from the file and return
+    if File.exists? file_path
+      return File.new(file_path).read if ((Time.now - File.mtime(file_path)) < max_age)
+    end
+
+    # if the file does not exist (or if the data is not fresh), we
+    #  make an get request and save it to a file
+    response_string = ""
+    response = get rest_url("metadata/get_metadata/#{file_name}")
+
+    if (response["status"] == "ok")
+      file = File.open(file_path, "w") do |data|
+        data << response_string = response["data"]
+      end
+    end
+
+    return response_string
+  end
+
+  def save_cookie(cookie_content)
+    file_path = File.join(@cache_dir, COOKIE_HOLDER_NAME)
+    File.open(file_path, "w") do |file|
+      Marshal.dump(cookie_content, file)
+    end
+  end
+
+  def load_cookie
+    file_path = File.join(@cache_dir, COOKIE_HOLDER_NAME)
+    cookie_content = File.exists?(file_path) ? File.open(file_path) {|f| Marshal.load(f)} : nil
+    cookie_content
+  end
+
+end

data/lib/configurator.rb

@@ -0,0 +1,176 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'rubygems'
+require 'fileutils'
+dtk_require_from_base('util/os_util')
+
+module DTK
+  module Client
+    class Configurator
+
+      CONFIG_FILE   = File.join(OsUtil.dtk_local_folder, "client.conf")
+      CRED_FILE     = File.join(OsUtil.dtk_local_folder, ".connection")
+      DIRECT_ACCESS = File.join(OsUtil.dtk_local_folder, ".add_direct_access")
+      NODE_SSH_CREDENTIALS = File.join(OsUtil.dtk_local_folder, "ssh_credentials.yaml")
+
+      FileUtils.mkdir(OsUtil.dtk_local_folder) unless File.directory?(OsUtil.dtk_local_folder)
+
+      def self.check_config_exists
+        exists = true
+        if !File.exists?(CONFIG_FILE)
+          puts "", "Please enter the DTK server address (example: instance.dtk.io)"
+          header = File.read(File.expand_path('../lib/config/client.conf.header', File.dirname(__FILE__)))
+          generate_conf_file(CONFIG_FILE, [['server_host', 'Server address']], header)
+          exists = false
+        end
+        if !File.exists?(CRED_FILE)
+          puts "", "Please enter your DTK login details"
+          generate_conf_file(CRED_FILE, [['username', 'Username'], ['password', 'Password']], '')
+          exists = false
+        end
+
+        exists
+      end
+
+      def self.check_git
+        if OsUtil.which('git') == nil
+          OsUtil.put_warning "[WARNING]", "Can't find the 'git' command in you path. Please make sure git is installed in order to use all features of DTK Client.", :yellow
+        else
+          OsUtil.put_warning "[WARNING]", 'Git username not set. This can cause issues while using DTK Client. To set it, run `git config --global user.name "User Name"`', :yellow if `git config --get user.name` == ""
+          OsUtil.put_warning "[WARNING]", 'Git email not set. This can cause issues while using DTK Client. To set it, run `git config --global user.email "me@here.com"`', :yellow if `git config --get user.email` == ""
+        end
+      end
+
+      # return true/false, .add_direct_access file location and ssk key file location
+      def self.check_direct_access
+        username_exists  = check_for_username_entry(client_username())
+        ssh_key_path = SSHUtil.default_rsa_pub_key_path()
+
+        {:username_exists => username_exists, :file_path => DIRECT_ACCESS, :ssh_key_path => ssh_key_path}
+      end
+
+      def self.generate_conf_file(file_path, properties, header)
+        require 'highline/import'
+        property_template = []
+
+        properties.each do |p,d|
+          begin
+            trap("INT") {
+              puts "", "Exiting..."
+              abort
+            }
+          end
+          value = ask("#{d}: ") { |q| q.echo = false if p == 'password'}
+          property_template << [p,value]
+        end
+
+        File.open(file_path, 'w') do |f|
+          f.puts(header)
+          property_template.each do |prop|
+            f.puts("#{prop[0]}=#{prop[1]}")
+          end
+        end
+      end
+
+      def self.regenerate_conf_file(file_path, properties, header)
+        File.open(file_path, 'w') do |f|
+          f.puts(header)
+          properties.each do |prop|
+            f.puts("#{prop[0]}=#{prop[1]}")
+          end
+        end
+      end
+
+      def self.create_missing_clone_dirs
+        FileUtils.mkdir(OsUtil.component_clone_location) unless File.directory?(OsUtil.component_clone_location)
+        FileUtils.mkdir(OsUtil.service_clone_location) unless File.directory?(OsUtil.service_clone_location)
+        FileUtils.mkdir(OsUtil.test_clone_location) unless File.directory?(OsUtil.test_clone_location)
+        FileUtils.mkdir(OsUtil.backups_location) unless File.directory?(OsUtil.backups_location)
+      end
+
+
+      def self.parse_key_value_file(file)
+        # adapted from mcollective config
+        ret = Hash.new
+        raise DTK::Client::DtkError,"Config file (#{file}) does not exists" unless File.exists?(file)
+        File.open(file).each do |line|
+          # strip blank spaces, tabs etc off the end of all lines
+          line.gsub!(/\s*$/, "")
+          unless line =~ /^#|^$/
+            if (line =~ /(.+?)\s*=\s*(.+)/)
+              key = $1
+              val = $2
+              ret[key.to_sym] = val
+            end
+          end
+        end
+        ret
+      end
+      def self.add_current_user_to_direct_access()
+        username = client_username()
+
+        File.open(DIRECT_ACCESS, 'a') do |file|
+          file.puts(username)
+        end
+
+        true
+      end
+
+      def self.client_username()
+        parse_key_value_file(CRED_FILE)[:username]
+      end
+
+      #
+      # Method will check if there is username entry in DIRECT_ACCESS file
+      #
+      def self.check_for_username_entry(username)
+        if File.exists?(DIRECT_ACCESS)
+          File.open(DIRECT_ACCESS).each do |line|
+            if line.strip.eql?(username)
+              return true
+            end
+          end
+        end
+
+        return false
+      end
+
+      def self.ask_catalog_credentials()
+        are_there_creds = Console.confirmation_prompt("Do you have DTK catalog credentials", true)
+        property_template = {}
+        if are_there_creds
+          property_template = self.enter_catalog_credentials()
+        end
+
+        property_template
+      end
+
+      def self.enter_catalog_credentials()
+        property_template = {}
+        # needed to preserve the order for ruby 1.8.7
+        # ruby 1.8 does not preserve order of insertation
+        wizard_values = { :username => 'Catalog Username', :password => 'Catalog Password' }
+        [:username, :password].each do |p|
+          value = ask("#{wizard_values[p]}: ") { |q| q.echo = false if p == :password }
+          property_template.store(p, value)
+        end
+        property_template
+      end
+    end
+  end
+end

data/lib/context_router.rb

@@ -0,0 +1,44 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK
+  module Client
+
+    #
+    # This class is used to reroute commands/tasks (Method invocations) from one context (Class) to another
+    #
+    class ContextRouter
+
+      extend DTK::Client::Auxiliary
+
+      # This method invokes target context task
+      def self.routeTask(target_context, target_method, target_context_params, conn)
+        target_context = target_context.to_s
+        target_method  = target_method.to_s
+
+        # Initing required params and invoking target_context.target_method
+        load_command(target_context)
+        target_context_class = DTK::Client.const_get "#{cap_form(target_context)}"
+
+        ret = target_context_class.execute_from_cli(conn, target_method, target_context_params, [], false)
+        ret.kind_of?(Response::NoOp) ? Response::Ok.new() : ret
+      end
+
+    end
+
+  end
+end

data/lib/core.rb

@@ -0,0 +1,497 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#TODO: user common utils in DTK::Common::Rest
+
+require 'rubygems'
+require 'singleton'
+require 'restclient'
+require 'colorize'
+require 'json'
+require 'pp'
+
+# Development Gems
+if ::DTK::Configuration.get(:development_mode)
+  require 'ap'
+  # require 'looksee'
+end
+
+#TODO: for testing; fix by pass in commadn line argument
+#RestClient.log = STDOUT
+
+dtk_require_from_base('domain/response')
+dtk_require_from_base('util/os_util')
+dtk_require_from_base('util/ssh_util')
+dtk_require_from_base('util/common_util')
+dtk_require_from_base('util/permission_util')
+dtk_require_from_base('util/remote_dependency_util')
+dtk_require_from_base('util/module_util')
+dtk_require_from_base('shell/message_queue')
+
+dtk_require("config/configuration")
+
+def top_level_execute(entity_name, method_name, context_params=nil, options_args=nil, shell_execute=false)
+  begin
+    top_level_execute_core(entity_name, method_name, context_params, options_args, shell_execute)
+  rescue DTK::Client::DtkLoginRequiredError
+    # re-logging user and repeating request
+    DTK::Client::OsUtil.print("Session expired: re-establishing session & repeating given task", :yellow)
+    DTK::Client::Session.re_initialize
+    top_level_execute_core(entity_name, method_name, context_params, options_args, shell_execute)
+  end
+end
+
+def top_level_execute_core(entity_name, method_name, context_params=nil, options_args=nil, shell_execute=false)
+  extend DTK::Client::OsUtil
+
+  entity_class = nil
+
+  begin
+    include DTK::Client::Auxiliary
+
+    entity_name = entity_name.gsub("-","_")
+    load_command(entity_name)
+    conn = DTK::Client::Session.get_connection
+
+    # if connection parameters are not set up properly then don't execute any command
+    return if validate_connection(conn)
+
+    # call proper thor class and task
+    entity_class = DTK::Client.const_get "#{cap_form(entity_name)}"
+
+    # call forwarding, in case there is no task for given entity we switch to last (n-context) and try than
+    unless (entity_class.task_names.include?(method_name))
+      entity_class = DTK::Client.const_get "#{cap_form(context_params.last_entity_name.to_s)}"
+    end
+
+    response_ruby_obj = entity_class.execute_from_cli(conn,method_name,context_params,options_args,shell_execute)
+
+    # it will raise DTK::Client::Error in case of error response
+    print_method_response!(response_ruby_obj)
+
+    # process/print queued message from server
+    DTK::Shell::MessageQueue.print_messages
+
+  rescue DTK::Client::DtkLoginRequiredError => e
+    # this error is handled in method above
+    raise e
+  rescue DTK::Client::DSLParsing => e
+    DTK::Client::OsUtil.print(e.message, :red)
+  rescue DTK::Client::DtkValidationError => e
+    validation_message = e.message
+
+    # if !e.skip_usage_info && entity_class && method_name
+    #   usage_info = entity_class.get_usage_info(entity_name, method_name)
+    #   validation_message += ", usage: #{usage_info}"
+    # end
+
+    if e.display_usage_info && entity_class && method_name
+      usage_info = entity_class.get_usage_info(entity_name, method_name)
+      validation_message += ", usage: #{usage_info}"
+
+      validation_message.gsub!("^^", '') if validation_message.include?("^^")
+      validation_message.gsub!("HIDE_FROM_BASE ", '') if validation_message.include?("HIDE_FROM_BASE")
+    end
+
+    DTK::Client::OsUtil.print(validation_message, :yellow)
+  rescue DTK::Client::DtkError => e
+    # this are expected application errors
+    DtkLogger.instance.error_pp(e.message, e.backtrace)
+  rescue Exception => e
+    client_internal_error = DTK::Client::DtkError::Client.label()
+    DtkLogger.instance.fatal_pp("[#{client_internal_error}] DTK has encountered an error #{e.class}: #{e.message}", e.backtrace)
+  end
+end
+
+def print_method_response!(response_ruby_obj)
+  # this will raise error if found
+  DTK::Client::ResponseErrorHandler.check(response_ruby_obj)
+
+  # this will find appropriate render adapter and give output, returns boolean
+  if print = response_ruby_obj.render_data
+    print = [print] unless print.kind_of?(Array)
+    print.each do |el|
+      if el.kind_of?(String)
+        el.each_line{|l| STDOUT << l}
+      else
+        PP.pp(el,STDOUT)
+      end
+    end
+  end
+end
+
+def load_command(command_name)
+  parser_adapter = DTK::Client::Config[:cli_parser] || "thor"
+
+  dtk_nested_require("parser/adapters",parser_adapter)
+  dtk_nested_require("commands/#{parser_adapter}",command_name)
+end
+
+# check if connection is set up properly
+def validate_connection(connection)
+  if connection.connection_error?
+    connection.print_warning
+    puts "\nDTK will now exit. Please set up your connection properly and try again."
+    return true
+  end
+
+  false
+end
+
+# check if .add_direct_access file exists, if not then add direct access and create .add_direct_access file
+def resolve_direct_access(params, config_exists=nil)
+  return if params[:username_exists]
+
+  puts "Processing ..." if config_exists
+  # check to see if catalog credentials are set
+  conn = DTK::Client::Session.get_connection
+  response = conn.post DTK::Client::CommandBase.class, conn.rest_url("account/check_catalog_credentials"), {}
+
+  # set catalog credentails
+  if response.ok? && !response.data['catalog_credentials_set']
+    # setting up catalog credentials
+    catalog_creds = DTK::Client::Configurator.ask_catalog_credentials
+    unless catalog_creds.empty?
+      response = conn.post DTK::Client::CommandBase.class, conn.rest_url("account/set_catalog_credentials"), { :username => catalog_creds[:username], :password => catalog_creds[:password], :validate => true}
+      if errors = response['errors']
+        DTK::Client::OsUtil.print("#{errors.first['message']} You will have to set catalog credentials manually ('dtk account set-catalog-credentials').", :yellow)
+      end
+    end
+  end
+
+  # response = DTK::Client::Account.add_access(params[:ssh_key_path])
+  response, matched_pub_key, matched_username = DTK::Client::Account.add_key(params[:ssh_key_path], true, "#{DTK::Client::Session.connection_username}-client")
+
+  if !response.ok?
+    DTK::Client::OsUtil.print("We were not able to add access for current user. #{response.error_message}. In order to properly use dtk-shell you will have to add access manually ('dtk account add-ssh-key').\n", :yellow)
+  elsif matched_pub_key
+    # message will be displayed by add key # TODO: Refactor this flow
+    DTK::Client::OsUtil.print("Provided SSH PUB key has already been added.", :yellow)
+    DTK::Client::Configurator.add_current_user_to_direct_access
+  elsif matched_username
+    DTK::Client::OsUtil.print("User with provided name already exists.", :yellow)
+  else
+    # commented out because 'add_key' method called above will also print the same message
+    # DTK::Client::OsUtil.print("Your SSH PUB key has been successfully added.", :yellow)
+    DTK::Client::Configurator.add_current_user_to_direct_access
+  end
+
+  response
+end
+
+module DTK
+  module Client
+    class ResponseErrorHandler
+      class << self
+
+        def check_for_session_expiried(response)
+          error_code = nil
+          if response && response['errors']
+            response['errors'].each do |err|
+              error_code      = err["code"]||(err["errors"] && err["errors"].first["code"])
+            end
+          end
+
+          return (error_code == "forbidden")
+        end
+
+        def check(response)
+          DtkError.raise_if_error?(response)
+        end
+      end
+    end
+
+    class Log
+      #TODO Stubs
+      def self.info(msg)
+        pp "info: #{msg}"
+      end
+      def self.error(msg)
+        pp "error: #{msg}"
+      end
+    end
+
+    module ParseFile
+
+      def parse_key_value_file(file)
+        DTK::Client::Configurator.parse_key_value_file(file)
+      end
+
+    end
+    class Config < Hash
+      include Singleton
+      include ParseFile
+      dtk_require_from_base('configurator')
+
+      CONFIG_FILE = ::DTK::Client::Configurator::CONFIG_FILE
+      CRED_FILE = ::DTK::Client::Configurator::CRED_FILE
+
+      REQUIRED_KEYS = [:server_host]
+
+      def self.[](k)
+        Config.instance[k]
+      end
+     private
+      def initialize
+        set_defaults
+        load_config_file
+        validate
+      end
+      def set_defaults
+        self[:server_port] = 80
+        self[:assembly_module_base_location] = 'assemblies'
+        self[:secure_connection] = true
+        self[:secure_connection_server_port] = 443
+      end
+
+      def load_config_file
+        parse_key_value_file(CONFIG_FILE).each{|k,v|self[k]=v}
+      end
+
+      def validate
+        #TODO: need to check for legal values
+        missing_keys = REQUIRED_KEYS - keys
+        raise DTK::Client::DtkError, "Missing config keys (#{missing_keys.join(",")}). Please check your configuration file #{CONFIG_FILE} for required keys!" unless missing_keys.empty?
+      end
+
+    end
+
+
+    ##
+    # Session Singleton we will use to hold connection instance, just a singleton wrapper.
+    # During shell input it will be needed only once, so singleton was obvious solution.
+    #
+    class Session
+      include Singleton
+
+      attr_accessor :conn
+
+      def initialize
+        @conn = DTK::Client::Conn.new
+      end
+
+      def self.get_connection
+        Session.instance.conn
+      end
+
+      def self.connection_username
+        Session.instance.conn.get_username
+      end
+
+      def self.re_initialize
+        Session.instance.conn = nil
+        Session.instance.conn = DTK::Client::Conn.new
+        Session.instance.conn.cookies
+      end
+
+      def self.logout
+        # from this point @conn is not valid, since there are no cookies set
+        Session.instance.conn.logout
+      end
+
+      # opts can have key
+      #   :command_class
+      def self.post(route, body, opts = {})
+        command_class = opts[:command_class] || Class
+        conn = Session.instance.conn
+        conn.post(command_class, conn.rest_url(route), body)
+      end
+    end
+
+    class Conn
+      def initialize
+        @cookies = Hash.new
+        @connection_error = nil
+        login
+      end
+
+      VERBOSE_MODE_ON = ::DTK::Configuration.get(:verbose_rest_calls)
+
+      attr_reader :connection_error, :cookies
+
+      if VERBOSE_MODE_ON
+        require 'ap'
+      end
+
+      def self.get_timeout
+        DefaultRestOpts[:timeout]
+      end
+
+      def self.set_timeout(timeout_sec)
+        DefaultRestOpts[:timeout] = timeout_sec
+      end
+
+      def get_username
+        get_credentials[:username]
+      end
+
+      def rest_url(route=nil)
+        protocol, port = "http", Config[:server_port].to_s
+        protocol, port = "https", Config[:secure_connection_server_port].to_s if Config[:secure_connection] == "true"
+
+        "#{protocol}://#{Config[:server_host]}:#{port}/rest/#{route}"
+      end
+
+      def get(command_class,url)
+        ap "GET #{url}" if VERBOSE_MODE_ON
+
+        check_and_wrap_response(command_class, Proc.new { json_parse_if_needed(get_raw(url)) })
+      end
+
+      def post(command_class,url,body=nil)
+        if VERBOSE_MODE_ON
+          ap "POST (REST) #{url}"
+          ap "params: "
+          ap body
+        end
+
+        check_and_wrap_response(command_class, Proc.new { json_parse_if_needed(post_raw(url,body)) })
+      end
+
+      def post_file(command_class,url,body=nil)
+        if VERBOSE_MODE_ON
+          ap "POST (FILE) #{url}"
+          ap "params: "
+          ap body
+        end
+
+        check_and_wrap_response(command_class, Proc.new { json_parse_if_needed(post_raw(url,body,{:content_type => 'avro/binary'})) })
+      end
+
+      # method will repeat request in case session has expired
+      def check_and_wrap_response(command_class, rest_method_func)
+        response = rest_method_func.call
+
+        if ResponseErrorHandler.check_for_session_expiried(response)
+          # re-logging user and repeating request
+          DTK::Client::OsUtil.print("Session expired: re-establishing session & re-trying request ...", :yellow)
+          @cookies = DTK::Client::Session.re_initialize
+          response = rest_method_func.call
+        end
+
+        response_obj = Response.new(command_class, response)
+
+        # queue messages from server to be displayed later
+        DTK::Shell::MessageQueue.process_response(response_obj)
+
+        response_obj
+      end
+
+
+
+      def connection_error?
+        return !@connection_error.nil?
+      end
+
+      def logout
+        response = get_raw rest_url("user/process_logout")
+
+        # save cookies - no need to persist them
+        # DiskCacher.new.save_cookie(@cookies)
+
+        raise DTK::Client::DtkError, "Failed to logout, and terminate session!" unless response
+        @cookies = nil
+      end
+
+      ##
+      # Method will warn user that connection could not be established. User should check configuration
+      # to make sure that connection is properly set.
+      #
+      def print_warning
+        creds = get_credentials
+        puts   "[WARNING] Unable to connect to server, please check you configuration."
+        puts   "========================== Configuration =========================="
+        printf "%15s %s\n", "REST endpoint:", rest_url
+        printf "%15s %s\n", "Username:", "#{creds[:username]}"
+        printf "%15s %s\n", "Password:", "#{creds[:password] ? creds[:password].gsub(/./,'*') : 'No password set'}"
+        puts   "==================================================================="
+
+        if self.connection_error['errors'].first['errors']
+          error_code = self.connection_error['errors'].first['errors'].first['code']
+          print " Error code: "
+          DTK::Client::OsUtil.print(error_code, :red)
+        end
+      end
+
+      private
+
+      include ParseFile
+
+      def login
+        creds = get_credentials
+        response = post_raw rest_url("user/process_login"),creds
+        errors = response['errors']
+
+        if response.kind_of?(Common::Response) and not response.ok?
+          if (errors && errors.first['code']=="pg_error")
+            DTK::Client::OsUtil.print(errors.first['message'].gsub!("403 Forbidden", "[PG_ERROR]"), :red)
+            exit
+          end
+          @connection_error = response
+        else
+          @cookies = response.cookies
+        end
+      end
+
+      def set_credentials(username, password)
+        @parsed_credentials = { :username => username, :password => password}
+      end
+
+      def get_credentials
+        unless @parsed_credentials
+          cred_file = Config::CRED_FILE
+          raise DTK::Client::DtkError,"Authorization configuration file (#{cred_file}) does not exist" unless File.exists?(cred_file)
+          ret = parse_key_value_file(cred_file)
+          [:username,:password].each{ |k| raise DTK::Client::DtkError, "cannot find #{k}" unless ret[k] }
+          @parsed_credentials = ret
+        end
+
+        @parsed_credentials
+      end
+
+      ####
+      RestClientWrapper = Common::Response::RestClientWrapper
+
+      # In development mode we want bigger timeout allowing us to debbug on server while still
+      # keeping connection alive and receivinga response
+      if ::DTK::Configuration.get(:development_mode)
+        DefaultRestOpts = {:timeout => 2000, :open_timeout => 10, :error_response_class => Client::Response::Error}
+        # DefaultRestOpts = {:timeout => 50, :open_timeout => 2, :error_response_class => Client::Response::Error}
+      else
+        DefaultRestOpts = {:timeout => 300, :open_timeout => 10, :error_response_class => Client::Response::Error}
+      end
+
+      # enable SSL verification
+      DefaultRestOpts.merge!(:verify_ssl => OpenSSL::SSL::VERIFY_PEER)
+      # Net:HTTP from Ruby 1.8.7 doesn't verify SSL certs correctly
+      # this is a CA bundle downloaded from http://curl.haxx.se/docs/caextract.html,
+      # and it will only be used for 1.8.7, otherwise the default (system) CA will be used
+      DefaultRestOpts.merge!(:ssl_ca_file => File.expand_path('../lib/config/cacert.pem', File.dirname(__FILE__)))
+
+      def get_raw(url)
+        RestClientWrapper.get_raw(url, {}, DefaultRestOpts.merge(:cookies => @cookies))
+      end
+      def post_raw(url,body,params={})
+        RestClientWrapper.post_raw(url, body, DefaultRestOpts.merge(:cookies => @cookies).merge(params))
+      end
+
+      def json_parse_if_needed(item)
+        RestClientWrapper.json_parse_if_needed(item)
+      end
+    end
+  end
+end