dtk-network-client 1.0.0

data/lib/client/git_repo.rb

@@ -0,0 +1,221 @@
+module DTK::Network::Client
+  class GitRepo
+    def self.add_remote_and_publish(git_args)
+      Command.wrap_command(git_args) do |git_args|
+        repo_dir      = git_args.required(:repo_dir)
+        remote_url    = git_args.required(:remote_url)
+        local_branch  = git_args[:branch] || 'master'
+        remote_branch = git_args[:remote_branch] || local_branch
+        remote        = git_args[:remote] || 'origin'
+
+        repo = git_repo.new(repo_dir, :branch => local_branch)
+        create_if_missing = local_branch_exist?(repo, local_branch) ? false : true
+
+        repo.checkout(local_branch, new_branch: create_if_missing)
+        repo.add_all
+        repo.commit("Publish from dtk client", :allow_empty => true)
+        add_remote_and_push(repo, remote, remote_url, remote_branch)
+      end
+    end
+
+    def self.push_to_remote(git_args)
+      Command.wrap_command(git_args) do |git_args|
+        repo_dir      = git_args.required(:repo_dir)
+        remote_url    = git_args.required(:remote_url)
+        local_branch  = git_args[:branch] || 'master'
+        remote_branch = git_args[:remote_branch] || local_branch
+        remote        = git_args[:remote] || 'origin'
+        commit_msg    = git_args[:commit_msg] || "Push from dtkn client"
+
+        repo = git_repo.new(repo_dir, :branch => local_branch)
+        create_if_missing = local_branch_exist?(repo, local_branch) ? false : git_args[:create_if_missing]
+
+        repo.checkout(local_branch, new_branch: create_if_missing)
+        repo.add_all
+        repo.commit(commit_msg, :allow_empty => true)
+
+        if repo.is_there_remote?(remote)
+          push_when_there_is_remote(repo, remote, remote_url, remote_branch)
+        else
+          add_remote_and_push(repo, remote, remote_url, remote_branch)
+        end
+      end
+    end
+
+    def self.pull_from_remote(git_args)
+      Command.wrap_command(git_args) do |git_args|
+        repo_dir      = git_args.required(:repo_dir)
+        remote_url    = git_args.required(:remote_url)
+        local_branch  = git_args[:branch] || 'master'
+        remote_branch = git_args[:remote_branch] || local_branch
+        remote        = git_args[:remote] || 'origin'
+
+        repo = git_repo.new(repo_dir, :branch => local_branch)
+        repo.checkout(local_branch)
+
+        if repo.is_there_remote?(remote)
+          pull_when_there_is_remote(repo, remote, remote_url, remote_branch)
+        else
+          add_remote_and_pull(repo, remote, remote_url, remote_branch)
+        end
+      end
+    end
+
+    # opts can have keys
+    #   :branch
+    # returns object of type DTK::Client::GitRepo
+    def self.create_empty_git_repo?(repo_dir, opts = {})
+      git_repo.new(repo_dir, :branch => opts[:branch])
+    end
+
+    # returns head_sha
+    def self.empty_commit(repo, commit_msg = nil)
+      repo.empty_commit(commit_msg)
+      repo.head_commit_sha
+    end
+
+    def self.add_remote(repo, remote_name, remote_url)
+      repo.add_remote(remote_name, remote_url)
+      remote_name
+    end
+
+    def self.fetch(repo, remote_name)
+      repo.fetch(remote_name)
+    end
+
+    def self.push_when_there_is_remote(repo, remote, remote_url, remote_branch)
+      repo.remove_remote(remote)
+      add_remote_and_push(repo, remote, remote_url, remote_branch)
+    end
+
+    def self.pull_when_there_is_remote(repo, remote, remote_url, remote_branch)
+      repo.remove_remote(remote)
+      add_remote_and_pull(repo, remote, remote_url, remote_branch)
+    end
+
+    def self.add_remote_and_push(repo, remote, remote_url, remote_branch)
+      repo.add_remote(remote, remote_url)
+      repo.push(remote, remote_branch)
+    end
+
+    def self.add_remote_and_pull(repo, remote, remote_url, remote_branch)
+      repo.add_remote(remote, remote_url)
+      repo.pull(remote, remote_branch)
+    end
+
+    def self.local_branch_exist?(repo, branch)
+      local_branches = branch.is_a?(String) ? repo.all_branches.local.map { |branch| branch.name } : (repo.all_branches.local || [])
+      local_branches.include?(branch)
+    end
+
+    # opts can have keys
+    #   :no_commit
+    def self.merge(repo, merge_from_ref, opts = {})
+      base_sha = repo.head_commit_sha
+      repo.merge(merge_from_ref, :use_theirs => opts[:use_theirs])
+      # the git gem does not take no_commit as merge argument; so doing it with soft reset
+      repo.reset_soft(base_sha) if opts[:no_commit]
+      repo.head_commit_sha
+    end
+
+    def self.local_ahead?(repo, merge_from_ref, opts = {})
+      base_sha = repo.head_commit_sha
+      remote_branch = repo.all_branches.remote.find { |r| "#{r.remote}/#{r.name}" == merge_from_ref }
+      remote_sha = remote_branch.gcommit.sha
+      repo.local_ahead(base_sha, remote_sha)
+    end
+
+    # opts can have keys:
+    #   :commit_msg
+    # returns head_sha
+    def self.stage_and_commit(repo_dir,local_branch_type, opts = {})
+      local_branch = branch_from_local_branch_type(local_branch_type)
+      repo = create_empty_git_repo?(repo_dir, :branch => local_branch)
+      repo.stage_and_commit(opts[:commit_msg])
+      repo.head_commit_sha
+    end
+
+    # TODO: DTK-2765: see what this does and subsume by create_add_remote_and_push
+    # For this and other methods in Internal that use Dtk_Server::GIT_REMOTE
+    # put a version in Internal taht takes remote_name as param and then have 
+    # # method with same name in Dtk, that calss this with appropriate remote name
+    # def self.init_and_push_from_existing_repo(repo_dir, repo_url, remote_branch)
+    #   repo = git_repo.new(repo_dir)
+
+    #   if repo.is_there_remote?(Dtk_Server::GIT_REMOTE)
+    #     push_when_there_is_dtk_remote(repo, repo_dir, repo_url, remote_branch)
+    #   else
+    #     add_remote_and_push(repo, repo_url, remote_branch)
+    #   end
+
+    #   repo.head_commit_sha
+    # end
+
+    def self.pull_from_remote(args)
+      repo_url       = args.required(:repo_url)
+      remote_branch  = args.required(:branch)
+      repo_dir       = args.required(:repo_dir)
+
+      repo = git_repo.new(repo_dir, :branch => remote_branch)
+      repo.pull(repo.remotes.first, remote_branch)
+    end
+
+    # def self.push_when_there_is_dtk_remote(repo, repo_dir, repo_url, remote_branch)
+    #   # if there is only one remote and it is dtk-server; remove .git and initialize and push as new repo to dtk-server remote
+    #   # else if multiple remotes and dtk-server being one of them; remove dtk-server; add new dtk-server remote and push
+    #   if repo.remotes.size == 1
+    #     git_repo.unlink_local_clone?(repo_dir)
+    #     create_repo_from_remote_and_push(repo_dir, repo_url, remote_branch)
+    #   else
+    #     repo.remove_remote(Dtk_Server::GIT_REMOTE)
+    #     add_remote_and_push(repo, repo_url, remote_branch)
+    #   end
+    # end
+
+    # def self.create_repo_from_server_remote(repo_dir, repo_url, remote_branch)
+    #   repo = git_repo.new(repo_dir, :branch => Dtkn::LOCAL_BRANCH)
+    #   repo.checkout(Dtkn::LOCAL_BRANCH, :new_branch => true)
+    #   repo.add_remote(Dtk_Server::GIT_REMOTE, repo_url)
+    #   repo
+    # end
+
+    # def self.create_repo_from_remote_and_push(repo_dir, repo_url, remote_branch)
+    #   repo = create_repo_from_server_remote(repo_dir, repo_url, remote_branch)
+    #   # repo = git_repo.new(repo_dir, :branch => Dtkn::LOCAL_BRANCH)
+    #   # repo.checkout(Dtkn::LOCAL_BRANCH, :new_branch => true)
+    #   # repo.add_remote(Dtk_Server::GIT_REMOTE, repo_url)
+    #   repo.stage_and_commit
+    #   repo.push(Dtk_Server::GIT_REMOTE, remote_branch, { :force => true })
+    #   repo.head_commit_sha
+    # end
+
+    # def self.add_remote_and_push(repo, repo_url, remote_branch)
+    #   repo.add_remote(Dtk_Server::GIT_REMOTE, repo_url)
+    #   repo.stage_and_commit
+    #   repo.push(Dtk_Server::GIT_REMOTE, remote_branch, { :force => true })
+    # end
+
+    def self.all_branches(args)
+      repo_url = args.required(:path)
+      repo = git_repo.new(repo_url)
+      repo.all_branches
+    end
+
+    def self.current_branch(args)
+      repo_url = args.required(:path)
+      repo = git_repo.new(repo_url)
+      repo.current_branch.name
+    end
+
+    def self.git_repo
+      GitClient
+    end
+
+    def self.reset_hard(repo, merge_from_ref)
+      repo.reset_hard(merge_from_ref)
+      repo.head_commit_sha
+    end
+
+  end
+end
+

data/lib/client/module_dir.rb

@@ -0,0 +1,38 @@
+require 'fileutils'
+
+module DTK::Network::Client
+  # Operations for managing module folders
+  class ModuleDir
+    extend DTK::Network::Client::Util::Tar
+
+    def self.ret_path_with_current_dir(name)
+      "#{Dir.getwd}/#{name.gsub(':','/')}"
+    end
+
+    def self.rm_f(path)
+      FileUtils.rm_rf(path)
+    end
+
+    def self.delete_directory_content(path)
+      FileUtils.rm_rf(Dir.glob("#{path}/*"))
+    end
+
+    def self.create_file_with_content(file_path, content)
+      FileUtils.mkdir_p(File.dirname(file_path))
+      File.open(file_path, 'w') { |f| f << content }
+    end
+
+    def self.create_and_ret_tar_gz(source_dir, opts = {})
+      raise Error.new("Directory '#{source_dir}' does not exist!") unless Dir.exist?(source_dir)
+      gzip(tar(source_dir, opts))
+    end
+
+    def self.ungzip_and_untar(file, target_dir)
+      raise Error.new("File '#{file}' does not exist!") unless File.exist?(file)
+      FileUtils.mkdir_p(target_dir)
+      untar(ungzip(File.open(file, "rb")), target_dir)
+    end
+
+  end
+end
+

data/lib/client/module_ref/dependency/local.rb

@@ -0,0 +1,47 @@
+module DTK::Network::Client
+  class ModuleRef
+    class Dependency
+      class Local < self
+        MODULE_FILE = 'dtk.module.yaml'
+
+        attr_reader :version, :source
+        def initialize(module_info)
+          super(name: module_info[:name] || module_info['module'], namespace: module_info[:namespace] || module_info['namespace'])
+          version_hash   = module_info[:version] || module_info['version']
+          version_str    = version_hash[:version] || version_hash['version']
+          version_source = version_hash[:source] || version_hash['source']
+
+          @version = ModuleRef::Version.new(version_str)
+          @source  = find_source(version_source)
+        end
+
+        def dtkn_versions_with_dependencies
+          require 'dtk_dsl'
+          file_type         = DTK::DSL::FileType::CommonModule::DSLFile::Top
+          file_obj          = DTK::DSL::FileObj.new(file_type, @source, { content: FileHelper.get_content?("#{@source}/#{MODULE_FILE}") })
+          parsed_module     = file_obj.parse_content(:common_module_summary)
+          dependent_modules = parsed_module.val(:DependentModules) || []
+
+          dependencies = dependent_modules.map { |dep| { 'namespace' => dep[:namespace], 'module' => dep[:module_name], 'version' => dep[:version] }}
+          [
+            {
+              'name' => self.version.str_version,
+              'version' => self.version.str_version,
+              'dependencies' => dependencies
+            }
+          ]
+        end
+
+        private
+
+        def find_source(version_source)
+          if matching_source = version_source.match(/(file:)(.*)/)
+            matching_source[2]
+          else
+            fail "Unsuppored source format: #{version_source}!"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/client/module_ref/dependency/remote.rb

@@ -0,0 +1,18 @@
+module DTK::Network::Client
+  class ModuleRef
+    class Dependency
+      class Remote < self
+        def initialize(module_info)
+          super(name: module_info[:name] || module_info['module'], namespace: module_info[:namespace] || module_info['namespace'])
+          version_str = module_info[:version]||module_info['version']
+          @version    = ModuleRef::Version.new(version_str)
+        end
+
+        def dtkn_versions_with_dependencies
+          response = rest_get("modules/get_versions_with_dependencies", { name: self.name, namespace: self.namespace })
+          response['versions']
+        end
+      end
+    end
+  end
+end

data/lib/client/module_ref/dependency.rb

@@ -0,0 +1,21 @@
+module DTK::Network::Client
+  class ModuleRef
+    class Dependency < self
+      require_relative('dependency/local')
+      require_relative('dependency/remote')
+
+      include RestWrapper
+      extend RestWrapper
+
+      def self.create_local_or_remote(module_info)
+        version = module_info[:version] || module_info['version']
+        is_local?(version) ? Local.new(module_info) : Remote.new(module_info)
+      end
+
+      def self.is_local?(version)
+        return unless version.is_a?(Hash)
+        !!(version[:source] || version['source'])
+      end
+    end
+  end
+end

data/lib/client/module_ref/version/source.rb

@@ -0,0 +1,18 @@
+module DTK::Network::Client
+  class ModuleRef
+    class Version
+      class Source
+        attr_reader :location
+        def initialize(source)
+          # right now we only support file as source, later we can introduce other sources
+          if matching_source = source.match(/(file:)(.*)/)
+            @location = matching_source[2]
+          else
+            fail "Unsuppored source format: #{source}!"
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/client/module_ref/version.rb

@@ -0,0 +1,87 @@
+module DTK::Network::Client
+  class ModuleRef
+    class Version
+      attr_reader :full_version, :requirement, :semantic_version, :str_version
+
+      def initialize(version = '')
+        @full_version     = version.strip
+        @str_version      = ''
+        @requirement      = '='
+        @semantic_version = nil
+        parse
+      end
+
+      def versions_in_range(versions)
+        v_in_range = []
+        versions   = [versions] unless versions.is_a?(Array)
+
+        versions.each do |version|
+          if satisfied_by?(version)
+            v_in_range << version
+          end
+        end
+
+        v_in_range
+      end
+
+      def satisfied_by?(version = nil)
+        return unless version
+        match_requirement?(version)
+      end
+
+      def is_semantic_version?
+        !!@semantic_version
+      end
+
+      def self.is_semantic_version?(version)
+        SemVer.parse(version)
+      end
+
+      private
+
+      def match_requirement?(version)
+        case @requirement
+        when '='
+          version == @str_version
+        when '<'
+          version < @str_version
+        when '>'
+          version > @str_version
+        when '<='
+          version <= @str_version
+        when '>='
+          version >= @str_version
+        when '~>'
+          top_version = nil
+
+          if patch = @semantic_version.patch
+            top_version = "#{@semantic_version.major}.#{@semantic_version.minor + 1}.0"
+          elsif minor = @semantic_version.minor
+            top_version = "#{@semantic_version.major + 1}.0.0"
+          end
+
+          return false unless top_version
+
+          (version < top_version) && (version >= @str_version)
+        else
+          false
+        end
+      end
+
+      def parse
+        parsed = @full_version.split(' ')
+        raise "Invalid version #{@full_version}!" if parsed.empty? || parsed.size > 2
+
+        if parsed.size == 1
+          @str_version = parsed.first
+          @semantic_version = SemVer.parse(@str_version)
+        else
+          @requirement      = parsed.first
+          @str_version      = parsed.last
+          @semantic_version = SemVer.parse(@str_version)
+        end
+      end
+    end
+  end
+end
+

data/lib/client/module_ref.rb

@@ -0,0 +1,21 @@
+module DTK::Network
+  module Client
+    class ModuleRef
+      require_relative('module_ref/dependency')
+      require_relative('module_ref/version')
+
+      attr_reader :name, :namespace, :version, :repo_dir, :full_name, :explicit_path
+
+      def initialize(module_info)
+        @name            = module_info[:name]
+        @namespace       = module_info[:namespace]
+        mod_info_version = module_info[:version] || module_info['version']
+        @version         = mod_info_version ? Version.new(mod_info_version) : nil
+        @repo_dir        = module_info[:repo_dir]
+        @explicit_path   = module_info[:explicit_path]
+        @full_name       = "#{@namespace}/#{@name}"
+      end
+    end
+  end
+end
+

data/lib/client/response/response_types.rb

@@ -0,0 +1,42 @@
+module DTK::Network::Client
+  class Response
+    class Ok < self
+      def initialize(data = {})
+        super('data'=> data, 'status' => 'ok')
+      end
+    end
+    
+    class NotOk < self
+      def initialize(data = {})
+        super('data'=> data, 'status' => 'notok')
+      end
+    end
+    
+    class NoOp < self
+      def render_data
+      end
+    end
+    
+    class ErrorResponse < self
+      include ::DTK::Common::Response::ErrorMixin
+      def initialize(hash = {})
+        super('errors' => [hash])
+      end
+      private :initialize
+      
+      class Usage < self
+        def initialize(hash_or_string = {})
+          hash = (hash_or_string.kind_of?(String) ? {'message' => hash_or_string} : hash_or_string)
+          super({'code' => 'error'}.merge(hash))
+        end
+      end
+      
+      class Internal < self
+        def initialize(hash = {})
+          super({'code' => 'error'}.merge(hash).merge('internal' => true))
+        end
+      end
+    end
+  end
+end
+

data/lib/client/response.rb

@@ -0,0 +1,27 @@
+require 'dtk_common_core' 
+
+module DTK::Network::Client
+  class Response < ::DTK::Common::Response
+    require_relative('response/response_types')
+
+    def initialize(hash = {})
+      super(hash)
+    end
+
+    def notok?
+      kind_of?(NotOk)
+    end
+
+    def self.wrap_as_response(data = {}, &block)
+      results = (block ? yield : data)
+      if results.nil?
+        NoOp.new
+      elsif results.kind_of?(Response)
+        results
+      else
+        Ok.new(results)
+      end
+    end
+  end
+end
+

data/lib/client/rest_wrapper.rb

@@ -0,0 +1,43 @@
+module DTK::Network::Client
+  module RestWrapper
+    def rest_get(url, params = {})
+      raise_error_if_notok_response do
+        Session.rest_get(url, params)
+      end
+    end
+
+    def rest_post(url, post_body = {})
+      raise_error_if_notok_response do
+        Session.rest_post(url, post_body)
+      end
+    end
+
+    def rest_delete(url, post_body = {})
+      raise_error_if_notok_response do
+        Session.rest_delete(url, post_body)
+      end
+    end
+
+    private
+
+    def raise_error_if_notok_response(&block)
+      response = block.call
+      if response
+        if response.is_a?(Hash)
+          status = response['status']
+          if status
+            raise Error.new(response) if status.eql?('notok')
+            # response
+          # else
+            # Response::Ok.new(response)
+          end
+          response
+        else
+          response
+        end
+      else
+        raise Error.new(response)
+      end
+    end
+  end
+end

data/lib/client/s3_helper.rb

@@ -0,0 +1,23 @@
+module DTK::Network::Client
+  class S3Helper
+    def self.ret_s3_bucket_info(response)
+      branch = response['branch'] || {}
+      bucket = nil
+      object_name = nil
+
+      if meta = branch['meta']
+        catalog_uri = meta['catalog_uri']
+        if match = catalog_uri.match(/.*amazonaws.com\/([^\/]*)\/(.*.gz)/)
+          bucket = match[1]
+          object_name = match[2]
+        end
+      else
+        raise "Unexpected that publish response does not contain branch metadata!"
+      end
+
+      raise "Unable to extract bucket and/or object name data from catalog_uri!" if bucket.nil? || object_name.nil?
+
+      return [bucket, object_name]
+    end
+  end
+end

data/lib/client/session.rb

@@ -0,0 +1,54 @@
+require 'singleton'
+
+module DTK::Network
+  module Client
+    class Session
+      include Singleton
+
+      attr_accessor :conn
+      
+      def initialize
+        @conn = Conn.new
+      end
+      
+      # opts can have keys
+      #  :reset
+      def self.get_connection(opts = {})
+        instance.conn = Conn.new if opts[:reset]
+        instance.conn
+      end
+      
+      # def self.connection_username
+      #   instance.conn.get_username
+      # end
+      
+      def self.re_initialize
+        instance.conn = nil
+        instance.conn = Conn.new
+        instance.conn.cookies
+      end
+      
+      def self.logout
+        # from this point @conn is not valid, since there are no cookies set
+        instance.conn.logout
+      end
+      
+      def self.rest_post(route, post_body = {})
+        instance.conn.post(route, post_body)
+      end
+
+      def self.rest_get(route, opts = {})
+        instance.conn.get(route, opts)
+      end
+
+      def self.rest_delete(route, delete_body = {})
+        instance.conn.delete(route, delete_body)
+      end
+
+      def self.get_codecommit_data
+        instance.conn.codecommit
+      end
+
+    end
+  end
+end

data/lib/client/storage/adapters/s3.rb

@@ -0,0 +1,25 @@
+module DTK::Network::Client
+  class Storage
+    module Adapter
+      class S3
+        require 'aws-sdk'
+
+        def initialize(data_hash)
+          @s3 = Aws::S3::Client.new(data_hash)
+        end
+
+        def upload(data_hash)
+          @s3.put_object(data_hash)
+        end
+
+        def download(data_hash, opts = {})
+          @s3.get_object(data_hash, opts)
+        end
+
+        def delete(data_hash, opts = {})
+          @s3.delete_object(data_hash, opts)
+        end
+      end
+    end
+  end
+end