dtk-network-client 1.0.0

data/lib/client/command/publish.rb

@@ -0,0 +1,120 @@
+module DTK::Network::Client
+  class Command
+    class Publish < self
+      def initialize(module_ref, dependency_tree, options = {})
+        @module_ref       = module_ref
+        @dependency_tree  = dependency_tree
+        @module_directory = module_ref.repo_dir
+        @options          = options
+        @parsed_module    = options[:parsed_module]
+        @development_mode = options[:development_mode]
+      end
+
+      def self.run(module_info, opts = {})
+        module_ref      = ModuleRef.new(module_info)
+        dependency_tree = DependencyTree.get_or_create(module_ref, opts.merge(save_to_file: true, development_mode: opts[:development_mode]))
+        new(module_ref, dependency_tree, opts).publish
+      end
+
+      def publish
+        module_info  = rest_post('modules', { name: @module_ref.name, namespace: @module_ref.namespace, return_if_exists: true })
+        dependencies = []
+
+        if @parsed_module
+          (@parsed_module.val(:DependentModules) || []).map do |parsed_mr|
+            dependencies << { 'namespace' => parsed_mr.req(:Namespace), 'module' => parsed_mr.req(:ModuleName), 'version' => parsed_mr.val(:ModuleVersion) }
+          end
+        end
+
+        if @development_mode
+          puts "Base module.yaml dependencies:\n#{dependencies}"
+        end
+
+        if @module_ref.version.is_semantic_version?
+          publish_semantic_version(module_info, dependencies)
+        else
+          publish_named_version(module_info, dependencies)
+        end
+      end
+
+      def publish_semantic_version(module_info, dependencies)
+        module_id          = module_info['id']
+        module_ref_version = @module_ref.version
+        branch             = rest_post("modules/#{module_id}/branch", { version: module_ref_version.str_version, dependencies: dependencies.to_json })
+        # repo_url           = ret_codecommit_url(module_info)
+
+        # git_init_and_publish_to_remote(branch['name'], repo_url)
+
+        published_response  = rest_post("modules/#{module_id}/publish", { version: module_ref_version.str_version })
+        bucket, object_name = S3Helper.ret_s3_bucket_info(published_response)
+        # gz_body             = ModuleDir.create_and_ret_tar_gz(@module_directory, exclude_git: true)
+
+        resource_name   = object_name.gsub('/','__')
+        published_creds = published_response['publish_credentails']
+        `tar -cpzf /tmp/#{resource_name} -C #{@module_directory} .`
+
+        require 'aws-sdk-s3'
+        s3_args = Args.new({
+          region: 'us-east-1',
+          access_key_id: published_creds['access_key_id'],
+          secret_access_key: published_creds['secret_access_key'],
+          session_token: published_creds['session_token']
+        })
+        s3 = Aws::S3::Resource.new(s3_args)
+        # storage = Storage.new(:s3, s3_args)
+
+        obj = s3.bucket(bucket).object(object_name)
+        obj.upload_file("/tmp/#{resource_name}")
+        FileUtils.remove_entry("/tmp/#{resource_name}")
+        # upload_args = Args.new({
+          # body: gz_body,
+          # bucket: bucket,
+          # key: object_name
+        # })
+        # storage.upload(upload_args)
+
+        rest_post("modules/update_status", { branch_id: branch['id'], status: 'published' })
+      end
+
+      def publish_named_version(module_info, dependencies)
+        module_id = module_info['id']
+        branch    = rest_post("modules/#{module_id}/branch", { version: @module_ref.version.str_version, dependencies: dependencies.to_json })
+        repo_url  = ret_codecommit_url(module_info)
+
+        git_init_and_publish_to_remote(branch['name'], repo_url)
+
+        rest_post("modules/update_status", { branch_id: branch['id'], status: 'published' })
+      end
+
+      def git_init_and_publish_to_remote(branch, repo_url)
+        git_args = Args.new({
+          repo_dir:   @module_directory,
+          branch:     branch,
+          remote_url: repo_url
+        })
+        GitRepo.add_remote_and_publish(git_args)
+      end
+
+      def ret_codecommit_url(module_info)
+        require 'open-uri'
+
+        if clone_url_http   = module_info['meta']['aws']['codecommit']['repository_metadata']['clone_url_http']
+          codecommit_data   = Session.get_codecommit_data
+          service_user_name = codecommit_data['service_specific_credential']['service_user_name']
+          service_password  = codecommit_data['service_specific_credential']['service_password']
+          encoded_password  = URI.encode_www_form_component(service_password)
+
+          url = nil
+          if match = clone_url_http.match(/^(https:\/\/)(.*)$/)
+            url = "#{match[1]}#{service_user_name}:#{encoded_password}@#{match[2]}"
+          end
+
+          url
+        else
+          raise "Unable to find codecommit https url"
+        end
+      end
+
+    end
+  end
+end

data/lib/client/command/pull.rb

@@ -0,0 +1,53 @@
+module DTK::Network::Client
+  class Command
+    class Pull < self
+      def initialize(module_ref, dependency_tree, options = {})
+        @module_ref       = module_ref
+        @dependency_tree  = dependency_tree
+        @module_directory = module_ref.repo_dir
+        @options          = options
+        @parsed_module    = options[:parsed_module]
+      end
+
+      def self.run(module_info, opts = {})
+        module_ref      = ModuleRef.new(module_info)
+        dependency_tree = DependencyTree.get_or_create(module_ref, opts.merge(save_to_file: true))
+        new(module_ref, dependency_tree, opts).pull
+      end
+
+      def pull
+        module_info = rest_get('modules/module_info', { name: @module_ref.name, namespace: @module_ref.namespace, version: @module_ref.version.str_version })
+        remote_url = construct_remote_url(module_info)
+        git_args = Args.new({
+          repo_dir:   @module_directory,
+          branch:     @module_ref.version.str_version,
+          remote_url: remote_url
+        })
+        GitRepo.pull_from_remote(git_args)
+      end
+
+      # TODO: move construct_remote_url to helper or mixin and use for all commands when needed
+      def construct_remote_url(module_info)
+        require 'open-uri'
+
+        # if clone_url_http = module_info.dig('meta', 'aws', 'codecommit', 'repository_metadata', 'clone_url_http')
+        if clone_url_http = module_info['meta']['aws']['codecommit']['repository_metadata']['clone_url_http']
+          codecommit_data   = Session.get_codecommit_data
+          # service_user_name = codecommit_data.dig('service_specific_credential', 'service_user_name')
+          # service_password  = codecommit_data.dig('service_specific_credential', 'service_password')
+          service_user_name = codecommit_data['service_specific_credential']['service_user_name']
+          service_password  = codecommit_data['service_specific_credential']['service_password']
+          encoded_password  = URI.encode_www_form_component(service_password)
+          url = nil
+          if match = clone_url_http.match(/^(https:\/\/)(.*)$/)
+            url = "#{match[1]}#{service_user_name}:#{encoded_password}@#{match[2]}"
+          end
+          url
+        else
+          raise "Unable to find codecommit https url"
+        end
+      end
+
+    end
+  end
+end

data/lib/client/command/push.rb

@@ -0,0 +1,63 @@
+module DTK::Network::Client
+  class Command
+    class Push < self
+      def initialize(module_ref, dependency_tree, options = {})
+        @module_ref       = module_ref
+        @dependency_tree  = dependency_tree
+        @module_directory = module_ref.repo_dir
+        @options          = options
+        @parsed_module    = options[:parsed_module]
+      end
+
+      def self.run(module_info, opts = {})
+        module_ref      = ModuleRef.new(module_info)
+        dependency_tree = DependencyTree.get_or_create(module_ref, opts.merge(save_to_file: true))
+        new(module_ref, dependency_tree, opts).push
+      end
+
+      def push
+        module_info = rest_get('modules/module_info', { name: @module_ref.name, namespace: @module_ref.namespace, version: @module_ref.version.str_version, module_action: 'push' })
+        remote_url = construct_remote_url(module_info)
+        git_args = Args.new({
+          repo_dir:   @module_directory,
+          branch:     @module_ref.version.str_version,
+          remote_url: remote_url
+        })
+        GitRepo.push_to_remote(git_args)
+
+        dependencies = []
+        if @parsed_module
+          (@parsed_module.val(:DependentModules) || []).map do |parsed_mr|
+            dependencies << { 'namespace' => parsed_mr.req(:Namespace), 'module' => parsed_mr.req(:ModuleName), 'version' => parsed_mr.val(:ModuleVersion) }
+          end
+        end
+
+        rest_post("modules/#{module_info['id']}/dependencies", { version: @module_ref.version.str_version, dependencies: dependencies.to_json })
+        nil
+      end
+
+      # TODO: move construct_remote_url to helper or mixin and use for all commands when needed
+      def construct_remote_url(module_info)
+        require 'open-uri'
+
+        # if clone_url_http = module_info.dig('meta', 'aws', 'codecommit', 'repository_metadata', 'clone_url_http')
+        if clone_url_http = module_info['meta']['aws']['codecommit']['repository_metadata']['clone_url_http']
+          codecommit_data   = Session.get_codecommit_data
+          # service_user_name = codecommit_data.dig('service_specific_credential', 'service_user_name')
+          service_user_name = codecommit_data['service_specific_credential']['service_user_name']
+          # service_password  = codecommit_data.dig('service_specific_credential', 'service_password')
+          service_password  = codecommit_data['service_specific_credential']['service_password']
+          encoded_password  = URI.encode_www_form_component(service_password)
+          url = nil
+          if match = clone_url_http.match(/^(https:\/\/)(.*)$/)
+            url = "#{match[1]}#{service_user_name}:#{encoded_password}@#{match[2]}"
+          end
+          url
+        else
+          raise "Unable to find codecommit https url"
+        end
+      end
+
+    end
+  end
+end

data/lib/client/command/remove_from_group.rb

@@ -0,0 +1,20 @@
+module DTK::Network::Client
+  class Command
+    class RemoveFromGroup < self
+      def initialize(group, user, options = {})
+        @group = group
+        @user      = user
+      end
+
+      def self.run(group, user, opts = {})
+        new(group, user, opts).revoke_access
+      end
+
+      def revoke_access
+        rest_post("groups/#{@group}/remove_member", { name: @group, username: @user })
+        nil
+      end
+
+    end
+  end
+end

data/lib/client/command/unpublish.rb

@@ -0,0 +1,27 @@
+module DTK::Network::Client
+  class Command
+    class Unpublish < self
+      def initialize(module_ref, options = {})
+        @module_ref       = module_ref
+        @options          = options
+      end
+
+      def self.run(module_info, opts = {})
+        module_ref      = ModuleRef.new(module_info)
+        new(module_ref, opts).unpublish
+      end
+
+      def unpublish
+        version = @module_ref.version
+        params = {
+          name: @module_ref.name,
+          namespace: @module_ref.namespace,
+          version: version.str_version
+        }
+        rest_post("modules/unpublish", params)
+
+        nil
+      end
+    end
+  end
+end

data/lib/client/command/update.rb

@@ -0,0 +1,18 @@
+module DTK::Network::Client
+  class Command
+    class Update < self
+      def initialize(module_ref, dependency_tree, options = {})
+        @module_ref       = module_ref
+        @dependency_tree  = dependency_tree
+        @module_directory = module_ref.repo_dir
+        @options          = options
+        @parsed_module    = options[:parsed_module]
+      end
+
+      def self.run(module_info, opts = {})
+        module_ref = ModuleRef.new(module_info)
+        DependencyTree.get_or_create(module_ref, opts.merge(save_to_file: true, update_lock_file: true))
+      end
+    end
+  end
+end

data/lib/client/command.rb

@@ -0,0 +1,31 @@
+module DTK::Network
+  module Client
+    class Command
+      require_relative('command/install')
+      require_relative('command/publish')
+      require_relative('command/list')
+      require_relative('command/info')
+      require_relative('command/push')
+      require_relative('command/pull')
+      require_relative('command/add_to_group')
+      require_relative('command/remove_from_group')
+      require_relative('command/delete')
+      require_relative('command/create_namespace')
+      require_relative('command/unpublish')
+      require_relative('command/update')
+      require_relative('command/chmod')
+      require_relative('command/delete_namespace')
+      require_relative('command/list_namespaces')
+
+      include RestWrapper
+      extend RestWrapper
+      include DTK::Network::Client::Util::Tar
+      extend DTK::Client::PermissionsUtil
+      include DTK::Client::PermissionsUtil
+
+      def self.wrap_command(args = Args.new, &block)
+        block.call(Args.convert(args))
+      end
+    end
+  end
+end

data/lib/client/config.rb

@@ -0,0 +1,54 @@
+require 'fileutils'
+
+module DTK::Network::Client
+  class Config
+    extend DTK::Network::Client::Util::OsUtil
+
+    DTK_NETWORK_FILE   = '.dtk_network'
+    DTK_NETWORK_CONFIG = File.join(dtk_local_folder, DTK_NETWORK_FILE)
+
+    def self.get_credentials
+      raise "Dtk network config file (#{DTK_NETWORK_CONFIG}) does not exist" unless File.exists?(DTK_NETWORK_CONFIG)
+      ret = parse_key_value_file(DTK_NETWORK_CONFIG)
+      [:email, :password].each{ |k| raise "cannot find #{k}" unless ret[k] }
+      {
+        email: ret[:email],
+        password: ret[:password]
+      }
+    end
+
+    def self.get_endpoint
+      raise "Dtk network config file (#{DTK_NETWORK_CONFIG}) does not exist" unless File.exists?(DTK_NETWORK_CONFIG)
+      ret = parse_key_value_file(DTK_NETWORK_CONFIG)
+      [:endpoint, :port].each{ |k| raise "cannot find #{k}" unless ret[k] }
+      "#{ret[:endpoint]}:#{ret[:port]}"
+    end
+
+    def self.module_download_location
+      if File.exists?(DTK_NETWORK_CONFIG)
+        ret = parse_key_value_file(DTK_NETWORK_CONFIG) || {}
+        ret[:download_location]
+      end
+    end
+    
+    def self.parse_key_value_file(file)
+      raise "Config file (#{file}) does not exists" unless File.exists?(file)
+
+      ret = Hash.new
+      File.open(file).each do |line|
+        # strip blank spaces, tabs etc off the end of all lines
+        line.gsub!(/\s*$/, "")
+        unless line =~ /^#|^$/
+          if (line =~ /(.+?)\s*=\s*(.+)/)
+            key = $1
+            val = $2
+            ret[key.to_sym] = val
+          end
+        end
+      end
+
+      ret
+    end
+
+  end
+end

data/lib/client/conn.rb

@@ -0,0 +1,132 @@
+module DTK::Network
+  module Client
+    class Conn
+      def initialize
+        @cookies          = {}
+        @connection_error = nil
+        @codecommit       = nil
+        login
+      end
+
+      attr_reader :cookies, :connection_error, :codecommit
+
+      def get(route, query_string_hash = {})
+        check_and_wrap_response { json_parse_if_needed(get_raw(rest_url(route), query_string_hash)) }
+      end
+
+      def post(route, post_body = {})
+        check_and_wrap_response { json_parse_if_needed(post_raw(rest_url(route), post_body)) }
+      end
+
+      def delete(route, delete_body = {})
+        check_and_wrap_response { json_parse_if_needed(delete_raw(rest_url(route), delete_body)) }
+      end
+
+      def connection_error?
+        !connection_error.nil?
+      end
+
+      private
+
+      def error_code?
+        connection_error['errors'].first['code'] rescue nil
+      end
+
+      REST_VERSION = 'v1'
+      REST_PREFIX = "api/#{REST_VERSION}"
+
+      def rest_url(route = nil)
+        "#{rest_url_base}/#{REST_PREFIX}/#{route}"
+      end
+
+      def rest_url_base
+        @@rest_url_base ||= Config.get_endpoint
+      end
+
+      def check_and_wrap_response(&rest_method_func)
+        if @connection_error
+          if conn_error = connection_error['errors'].first
+            raise Error, conn_error
+          else
+            raise Error, "Unable to connect to dtk network, please check your credentials and try again!"
+          end
+        end
+
+        response = rest_method_func.call
+
+        # response
+
+        # if Response::ErrorHandler.check_for_session_expiried(response)
+        #   # re-logging user and repeating request
+        #   OsUtil.print_warning("Session expired: re-establishing session & re-trying request ...")
+        #   @cookies = Session.re_initialize
+        #   response = rest_method_func.call
+        # end
+
+
+        # response_obj = Response.new(response)
+
+        # queue messages from server to be displayed later
+        #TODO: DTK-2554: put in processing of messages Shell::MessageQueue.process_response(response_obj)
+        # response_obj
+      end
+
+      def login
+        response = post_raw rest_url('auth/sign_in'), get_credentials
+        if response.kind_of?(::DTK::Common::Response) and !response.ok?
+          @connection_error = response
+        else
+          @cookies = response.cookies
+          set_codecommit_info(response)
+        end
+      end
+
+      def logout
+        response = get_raw rest_url('auth/sign_out')
+        # TODO: see if response can be nil
+        raise Error, "Failed to logout, and terminate session!" unless response
+        @cookies = nil
+      end
+
+      def get_credentials
+        @parsed_credentials ||= Config.get_credentials
+      end
+
+      def default_rest_opts
+        @default_rest_opts ||= get_default_rest_opts
+      end
+
+      def get_default_rest_opts
+        {
+          :timeout => 200,
+          :open_timeout => 10,
+          :verify_ssl => OpenSSL::SSL::VERIFY_PEER
+        }
+      end
+
+      def get_raw(url, query_string_hash = {})
+        Response::RestClientWrapper.get_raw(url, query_string_hash, default_rest_opts.merge(:cookies => @cookies))
+      end
+
+      def post_raw(url, post_body, params = {})
+        Response::RestClientWrapper.post_raw(url, post_body, default_rest_opts.merge(:cookies => @cookies).merge(params))
+      end
+
+      def delete_raw(url, delete_body, params = {})
+        Response::RestClientWrapper.delete_raw(url, delete_body, default_rest_opts.merge(:cookies => @cookies).merge(params))
+      end
+
+      def json_parse_if_needed(item)
+        Response::RestClientWrapper.json_parse_if_needed(item)
+      end
+
+      def set_codecommit_info(response)
+        json_response = json_parse_if_needed(response)
+        # if codecommit_data = json_response.dig('data', 'meta', 'aws', 'codecommit')
+        if codecommit_data = json_response['data']['meta']['aws']['codecommit']
+          @codecommit = codecommit_data
+        end
+      end
+    end
+  end
+end

data/lib/client/dependency_tree/activated.rb

@@ -0,0 +1,35 @@
+module DTK::Network
+  module Client
+    class DependencyTree
+      class Activated < Hash
+        def module_activated?(dependency)
+          if existing_dep = self["#{dependency.full_name}"]
+            if required_version = dependency.version
+              required_version.satisfied_by?(existing_dep[:version] || existing_dep['version'])
+            end
+          end
+        end
+
+        def add!(dependency_mod)
+          self.merge!("#{dependency_mod.full_name}" => generate_content(dependency_mod))
+        end
+
+        def existing_name?(name)
+          self[name]
+        end
+
+        def delete!(dependency_mod)
+          self.delete(dependency_mod.full_name)
+        end
+
+        def generate_content(dependency_mod)
+          if dependency_mod.respond_to?(:source)
+            { 'version' => dependency_mod.version.str_version, 'source' => dependency_mod.source }
+          else
+            { 'version' => dependency_mod.version.str_version }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/client/dependency_tree/cache.rb

@@ -0,0 +1,28 @@
+module DTK::Network::Client
+  class DependencyTree
+    class Cache < ::Hash
+      def initialize
+        super()
+      end
+
+      def add!(module_ref, dependencies)
+        self[index(module_ref)] ||= {:module_ref => module_ref, :dependencies => dependencies }
+      end
+
+      def lookup_dependencies?(module_ref)
+        (self[index(module_ref)] || {})[:dependencies]
+      end
+
+      def all_modules_refs
+        values.map { |hash| hash[:module_ref] }
+      end
+
+      private
+
+      def index(module_ref)
+        "#{module_ref.module_name}--#{module_ref.namespace}--#{module_ref.version.str_version}"
+      end
+
+    end
+  end
+end

data/lib/client/dependency_tree/candidates.rb

@@ -0,0 +1,19 @@
+module DTK::Network
+  module Client
+    class DependencyTree
+      class Candidates < Hash
+        def add!(dependency_mod, versions)
+          self.merge!(dependency_mod.full_name => { 'dependency_obj' => dependency_mod, 'versions' => versions })
+        end
+
+        # def existing_name?(name)
+        #   self[name]
+        # end
+
+        # def delete!(dependency_mod)
+        #   self.delete(dependency_mod.full_name)
+        # end
+      end
+    end
+  end
+end

data/lib/client/dependency_tree/resolver.rb

@@ -0,0 +1,16 @@
+module DTK::Network
+  module Client
+    class DependencyTree
+      class Resolver
+        def initialize(activated, dependencies)
+          @children = activated[:children]
+          @dependencies = dependencies
+        end
+
+        def resolve
+          
+        end
+      end
+    end
+  end
+end