dtk-network-client 1.0.0

data/lib/client/dependency_tree.rb

@@ -0,0 +1,272 @@
+require 'semverly'
+
+module DTK::Network
+  module Client
+    class DependencyTree
+      require_relative('dependency_tree/activated')
+      require_relative('dependency_tree/cache')
+      require_relative('dependency_tree/candidates')
+
+      include RestWrapper
+      extend RestWrapper
+
+      LOCK_FILE = "dtk.module.lock"
+
+      def initialize(module_ref, opts = {})
+        @module_ref       = module_ref
+        @module_directory = opts[:module_directory] || module_ref.repo_dir
+        @parsed_module    = opts[:parsed_module]
+        @cache            = Cache.new
+        @activated        = Activated.new
+        @candidates       = Candidates.new
+        @development_mode = opts[:development_mode]
+      end
+
+      def self.get_or_create(module_ref, opts = {})
+        content          = nil
+        module_ref       = convert_to_module_ref(module_ref) unless module_ref.is_a?(ModuleRef)
+        update_lock_file = opts[:update_lock_file]
+        yaml_content     = FileHelper.get_content?("#{module_ref.repo_dir}/#{LOCK_FILE}")
+
+        if yaml_content && !update_lock_file
+          content = YAML.load(yaml_content)
+          raise_error_if_dependencies_changed!(content, opts[:parsed_module])
+        elsif opts[:save_to_file] || update_lock_file
+          content = compute_and_save_to_file(module_ref, opts)
+        else
+          content = compute(module_ref, opts)
+        end
+
+        ret_required_format(content, opts[:format])
+      end
+
+      def self.compute_and_save_to_file(module_ref, opts = {})
+        activated = compute(module_ref, opts)
+        ModuleDir.create_file_with_content("#{module_ref.repo_dir}/#{LOCK_FILE}", YAML.dump(activated.to_h))
+        activated
+      end
+
+      def self.compute(module_ref, opts = {})
+        new(module_ref, opts).compute
+      end
+
+      def compute
+        dependencies = (ret_dependencies || []).map do |pm_ref|
+          ModuleRef::Dependency.create_local_or_remote(pm_ref)
+          # ModuleRef::Dependency.new({ name: pm_ref['module'], namespace: pm_ref['namespace'], version: pm_ref['version'] })
+        end
+
+        activate_dependencies(dependencies)
+        @activated
+      end
+
+      private
+
+      def activate_dependencies(dependencies, opts = {})
+        dependencies.each do |dependency|
+          next if @activated.module_activated?(dependency)
+
+          puts "Calculating dependencies for module: #{dependency.full_name}(#{dependency.version.str_version})" if @development_mode
+
+          check_for_conflicts(dependency)
+
+          # dtkn_versions_w_deps_hash = dtkn_versions_with_dependencies(dependency)
+          dtkn_versions_w_deps_hash = dependency.dtkn_versions_with_dependencies
+          dtkn_versions_w_deps = dtkn_versions_w_deps_hash.map { |v| v['version'] }
+
+          version_obj = dependency.version
+          versions_in_range = version_obj.versions_in_range(dtkn_versions_w_deps)
+
+          raise "No version matching requirement '#{version_obj.full_version}'" if versions_in_range.empty?
+
+          versions_in_range.sort!
+          latest_version = versions_in_range.last
+
+          if dependency.is_a?(ModuleRef::Dependency::Local)
+            latest_version_dep = dependency
+          else
+            latest_version_dep = ModuleRef::Dependency::Remote.new({ name: dependency.name, namespace: dependency.namespace, version: latest_version })
+          end
+
+          @activated.add!(latest_version_dep)
+          @candidates.add!(dependency, versions_in_range)
+
+          dtkn_deps_of_deps = dtkn_versions_w_deps_hash.find {|dep| dep['version'].eql?(latest_version) }
+          add_nested_modules(dependency, dtkn_deps_of_deps)
+
+          dtkn_deps_of_deps_objs = (dtkn_deps_of_deps['dependencies'] || {}).map do |dtkn_dep|
+            ModuleRef::Dependency.create_local_or_remote({ name: dtkn_dep['module'], namespace: dtkn_dep['namespace'], version: dtkn_dep['version'] })
+          end
+
+          activate_dependencies(dtkn_deps_of_deps_objs, opts)
+        end
+      end
+
+      def ret_dependencies
+        if @parsed_module
+          (@parsed_module.val(:DependentModules) || []).map do |parsed_mr|
+            { 'namespace' => parsed_mr.req(:Namespace), 'module' => parsed_mr.req(:ModuleName), 'version' => parsed_mr.val(:ModuleVersion) }
+          end
+        else
+          response = rest_get("modules/#{@module_ref.name}/dependencies", { name: @module_ref.name, namespace: @module_ref.namespace, version: @module_ref.version.str_version })
+          response['dependencies']
+        end
+      end
+
+      def dtkn_versions_with_dependencies(module_ref)
+        response = rest_get("modules/get_versions_with_dependencies", { name: module_ref.name, namespace: module_ref.namespace })
+        response['versions']
+      end
+
+      def check_for_conflicts(dependency)
+        if activated_mod = @activated.existing_name?(dependency.full_name)
+          raise "There is already activated version '#{activated_mod[:version] || activated_mod['version']}' for module '#{dependency.full_name}' and it does not match required version '#{dependency.version.full_version}'"
+        end
+      end
+
+      def add_nested_modules(dependency, dtkn_deps_of_deps)
+        if activated_module = @activated[dependency.full_name]
+          nested_deps = {}
+          (dtkn_deps_of_deps['dependencies'] || []).each{ |dtkn_dep| nested_deps.merge!("#{dtkn_dep['namespace']}/#{dtkn_dep['module']}" => simplify_version(dtkn_dep['version'])) }
+
+          unless nested_deps.empty?
+            activated_module.key?('modules') ? activated_module['modules'].merge!(nested_deps) : activated_module.merge!('modules' => nested_deps)
+          end
+        end
+      end
+
+      def simplify_version(version)
+        if version.is_a?(Hash)
+          # transform custom dtk hash into ruby hash to avoid strange output in yaml file
+          version.to_h
+        else
+          version
+        end
+      end
+
+      def self.ret_as_module_refs(dep_modules)
+        dep_modules.map { |k,v| create_module_ref(k, v) }
+      end
+
+      def self.create_module_ref(full_name, version_hash)
+        namespace, name = full_name.split('/')
+        version = version_hash[:version] || version_hash['version']
+        ModuleRef.new({ namespace: namespace, name: name, version: version })
+      end
+
+      def self.ret_as_hash(dep_modules)
+        dep_modules.map { |k,v| create_module_hash(k, v) }
+      end
+
+      def self.create_module_hash(full_name, version_hash)
+        namespace, name = full_name.split('/')
+        version = version_hash[:version] || version_hash['version']
+        ret = { namespace: namespace, name: name, version: version }
+
+        if source = version_hash[:source] || version_hash['source']
+          ret.merge!(source: source)
+        end
+
+        ret
+      end
+
+      def self.ret_required_format(content, format)
+        format ||= :module_ref
+        case format.to_sym
+        when :module_ref
+          ret_as_module_refs(content)
+        when :hash
+          ret_as_hash(content)
+        else
+          raise Error.new("Unsupported format '#{format}'. Valid formats are '#{ValidFormats.join(', ')}'")
+        end
+      end
+      ValidFormats = [:module_ref, :hash]
+
+      def self.convert_to_module_ref(module_ref)
+        ModuleRef.new(module_ref)
+      end
+
+      def self.raise_error_if_dependencies_changed!(lock_content, parsed_module)
+        return unless parsed_module
+
+        message      = "Module dependencies have been changed, please use '-u' option to update '#{LOCK_FILE}' file accordingly."
+        dependencies = convert_to_modules_lock_format(parsed_module.val(:DependentModules))
+
+        lock_deps = filter_out_dependencies_of_dependencies(lock_content, dependencies)
+        raise Error.new(message) if dependencies.size != lock_deps.size
+
+        dependencies.each do |name, value|
+          if matching = lock_deps[name]
+            module_ref_version = ModuleRef::Version.new(value['version'])
+            matches_version = module_ref_version.satisfied_by?(matching['version'])
+            matches_source = (value['source'] == matching['source'])
+            raise Error.new(message) unless (matches_version && matches_source)
+          else
+            raise Error.new(message)
+          end
+        end
+      end
+
+      def self.convert_to_modules_lock_format(dependencies = {})
+        modules_hash = {}
+
+        (dependencies || {}).each do |dependency|
+          version = dependency[:version]
+          source  = nil
+
+          if version.is_a?(Hash)
+            source = version['source'] || version[:source]
+            if matching_source = source && source.match(/(file:)(.*)/)
+              source = matching_source[2]
+            end
+            version = version['version'] || version[:version]
+          end
+
+          namespace_name = "#{dependency[:namespace]}/#{dependency[:module_name]}"
+          content_hash = {'version' => version}
+          content_hash['source'] = source if source
+          modules_hash.merge!(namespace_name => content_hash)
+        end
+
+        modules_hash
+      end
+
+      def self.filter_out_dependencies_of_dependencies(lock_content, module_yaml_dependencies)
+        top_level_dependencies = {}
+        nested_dependencies    = {}
+
+        lock_content.each do |lock_name, lock_value|
+          nested_dependencies.merge!(lock_value['modules'] || lock_value[:modules] || {})
+          top_level_dependencies.merge!(lock_name => {'version' => lock_value['version'], 'source' => lock_value['source']})
+        end
+
+        nested_dependencies.each do |nd_name, version|
+          source = nil
+
+          if version.is_a?(Hash)
+            source  = version['source']  || version[:source]
+            version = version['version'] || version[:version]
+          end
+
+          if top_level_dependency = top_level_dependencies[nd_name]
+            module_ref_version = ModuleRef::Version.new(version)
+            matches_version    = module_ref_version.satisfied_by?(top_level_dependency['version'])
+            matches_source     = (source == top_level_dependency['source'])
+
+            if yaml_dep = module_yaml_dependencies[nd_name]
+              matches_version_new = module_ref_version.satisfied_by?(yaml_dep['version'])
+              matches_source_new  = (source == yaml_dep['source'])
+              next unless (matches_version_new && matches_source_new)
+            end
+
+            top_level_dependencies.delete(nd_name) if (matches_version && matches_source)
+          end
+        end
+
+        top_level_dependencies
+      end
+
+    end
+  end
+end

data/lib/client/error.rb

@@ -0,0 +1,22 @@
+module DTK::Network::Client
+  class Error < NameError
+    def initialize(msg = '')
+      if errors = msg['errors']
+        errors = [errors] unless errors.is_a?(Array)
+        error_msg = ''
+        # error_msg << "#{errors['code'].upcase} " if errors['code']
+        errors.each do |error|
+          if err_msg = error['message']
+            error_msg << "#{err_msg}\n"
+          elsif orig_exeption = error['original_exception']
+            error_msg << "#{orig_exeption}\n"
+          end
+        end
+        super(error_msg)
+      else
+        super(msg)
+      end
+    end
+  end
+end
+

data/lib/client/file_helper.rb

@@ -0,0 +1,8 @@
+module DTK::Network::Client
+  class FileHelper
+    def self.get_content?(file_path)
+      File.open(file_path).read if file_path && File.exists?(file_path)
+    end
+  end
+end
+

data/lib/client/git_adapter/git_gem.rb

@@ -0,0 +1,246 @@
+require 'git'
+
+module DTK::Network
+  module Client
+    class GitGem
+      attr_accessor :git_repo
+
+      # Monkey patch the git merge method
+      # to allow passing the --allow-unrelated-histories flag
+      # more info: http://stackoverflow.com/questions/37937984/git-refusing-to-merge-unrelated-histories
+      class ::Git::Base
+        def merge(branch, message = 'merge', opts = {})
+          self.lib.merge(branch, message, opts)
+        end
+
+        def rev_list(sha)
+          self.lib.rev_list(sha)
+        end
+
+        def name_status(branch1 = nil, branch2 = nil, opts = {})
+          self.lib.name_status(branch1, branch2, opts)
+        end
+      end
+
+      class ::Git::Lib
+        def merge(branch, message = nil, opts = {})
+          arr_opts = []
+          arr_opts << '--allow-unrelated-histories' if opts[:allow_unrelated_histories]
+          arr_opts << '-Xtheirs' if opts[:use_theirs]
+          arr_opts << '-m' << message if message
+          arr_opts += [branch]
+          command('merge', arr_opts)
+        end
+
+        def rev_list(sha)
+          arr_opts = [sha]
+          command('rev-list', arr_opts)
+        end
+
+        def name_status(branch1 = nil, branch2 = nil, opts = {})
+          arr_opts = []
+          arr_opts << '--name-status'
+          arr_opts << "--diff-filter=#{opts[:diff_filter]}" if opts[:diff_filter]
+          arr_opts << branch1 if branch1
+          arr_opts << branch2 if branch2
+
+          command_lines('diff', arr_opts).inject({}) do |memo, line|
+            status, path = line.split("\t")
+            memo[path] = status
+            memo
+          end
+        end
+
+        def command_lines(cmd, opts = [], chdir = true, redirect = '')
+          cmd_op = command(cmd, opts, chdir)
+          op = cmd_op.encode("UTF-8", "binary", {
+                :invalid => :replace,
+                :undef => :replace
+              })
+          op.split("\n")
+        end
+      end
+
+      # opts can have keys
+      #  :branch
+      def initialize(repo_dir, opts = {})
+        @repo_dir          = repo_dir
+        @git_repo          = ::Git.init(repo_dir)
+        @local_branch_name = opts[:branch]
+      end
+
+      def self.clone(repo_url, target_path, branch)
+        git_base = ::Git.clone(repo_url, target_path)
+        begin
+          git_base.checkout(branch)
+        rescue => e
+          raise "The branch or tag '#{branch}' does not exist on repo '#{repo_url}'"
+        end
+        git_base
+      end
+
+      # opts can have keys
+      #  :new_branch - Boolean
+      def checkout(branch, opts = {})
+        ret = @git_repo.checkout(branch, opts)
+        @local_branch_name = branch
+        ret
+      end
+
+      def fetch(remote = 'origin')
+        @git_repo.fetch(remote)
+      end
+
+      def add_remote(name, url)
+        @git_repo.remove_remote(name) if is_there_remote?(name)
+        @git_repo.add_remote(name, url)
+      end
+
+      def remove_remote(name)
+        @git_repo.remove_remote(name) if is_there_remote?(name)
+      end
+
+      def push(remote, branch, opts = {})
+        branch_name = current_branch ? current_branch.name : 'master'
+        branch_for_push = "#{branch_name}:refs/heads/#{branch || local_branch_name}"
+        @git_repo.push(remote, branch_for_push, opts)
+      end
+
+      def merge(branch_to_merge_from, opts = {})
+        @git_repo.merge(branch_to_merge_from, 'merge', :allow_unrelated_histories => allow_unrelated_histories?, :use_theirs => opts[:use_theirs])
+      end
+
+      def status
+        @git_repo.status
+      end
+
+      def changed
+        status.is_a?(Hash) ? status.changed().keys : status.changed().collect { |file| file.first }
+      end
+
+      def untracked
+        status.is_a?(Hash) ? status.untracked().keys : status.untracked().collect { |file| file.first }
+      end
+
+      def deleted
+        status.is_a?(Hash) ? status.deleted().keys : status.deleted().collect { |file| file.first }
+      end
+
+      def added
+        status.is_a?(Hash) ? status.added().keys : status.added().collect { |file| file.first }
+      end
+
+      def stage_and_commit(commit_msg = nil)
+        commit_msg ||= default_commit_message
+        add_all
+        begin
+          commit(commit_msg)
+        rescue
+          # do not raise if nothing to commit
+        end
+      end
+
+      def empty_commit(commit_msg = nil)
+        commit_msg ||= default_commit_message
+        commit(commit_msg, :allow_empty => true)
+      end
+
+      def reset_soft(sha)
+        @git_repo.reset(sha)
+      end
+
+      def reset_hard(sha)
+        @git_repo.reset_hard(sha)
+      end
+
+      def revparse(sha_or_string)
+        @git_repo.revparse(sha_or_string)
+      end
+
+      def rev_list(base_sha)
+        @git_repo.rev_list(base_sha)
+      end
+
+      def local_ahead(base_sha, remote_sha)
+        results = @git_repo.rev_list(base_sha)
+        !results.split("\n").grep(remote_sha).empty?
+      end
+
+      def stage_changes()
+        @git_repo.add(untracked())
+        @git_repo.add(added())
+        @git_repo.add(changed())
+
+        deleted().each do |file|
+          begin
+            @git_repo.remove(file)
+          rescue
+            # ignore this error means file has already been staged
+            # we cannot support status of file, in 1.8.7 so this is
+            # solution for that
+          end
+        end
+      end
+
+      # opts can have keys
+      #   :allow_empty
+      def commit(commit_msg = "", opts = {})
+        @git_repo.commit(commit_msg, :allow_empty => opts[:allow_empty])
+      end
+
+      def add(*files)
+        @git_repo.add(files.flatten)
+      end
+
+      def add_all
+        # Cannot use '@git_repo.add(:all => true)' because this only works if pwd is base git repo
+        fully_qualified_repo_dir = (@repo_dir =~ /^\// ? @repo_dir : File.join(Dir.pwd, @repo_dir))
+        @git_repo.add(fully_qualified_repo_dir, :all => true )
+      end
+
+      def is_there_remote?(remote_name)
+        @git_repo.remotes.find { |r| r.name == remote_name }
+      end
+
+      def current_branch
+        @git_repo.branches.local.find { |b| b.current }
+      end
+
+      def remotes
+        @git_repo.remotes
+      end
+
+      def head_commit_sha
+        current_branch.gcommit.sha
+      end
+
+      def pull(remote, branch)
+        @git_repo.pull(remote, branch)
+      end
+
+      def diff
+        @git_repo.diff
+      end
+
+      def diff_name_status(branch_or_sha_1, branch_or_sha_2, opts = {})
+        @git_repo.name_status(branch_or_sha_1, branch_or_sha_2, opts)
+      end
+
+      def changed?
+        (!(changed().empty? && untracked().empty? && deleted().empty?))
+      end
+
+
+      def all_branches
+        @git_repo.branches
+      end
+
+      private
+
+      def default_commit_message
+        "DTK Commit from network client"
+      end
+
+    end
+  end
+end

data/lib/client/git_client.rb

@@ -0,0 +1,136 @@
+module DTK::Network
+  module Client
+    class GitClient
+      require_relative('git_adapter/git_gem')
+
+      # opts can have keys
+      #  :branch
+      def initialize(repo_dir, opts = {})
+        @repo_dir    = repo_dir
+        @git_adapter = git_adapter_class.new(repo_dir, opts)
+      end
+
+      attr_reader :repo_dir
+
+      def self.clone(repo_url, target_path, branch)
+        git_adapter_class.clone(repo_url, target_path, branch)
+      end
+
+      def self.is_git_repo?(dir)
+        File.directory?("#{dir}/.git")
+      end
+
+      def add_remote(name, url)
+        @git_adapter.add_remote(name, url)
+      end
+
+      def changed?
+        @git_adapter.changed?
+      end
+
+      # opts can have keys
+      #  :new_branch - Boolean
+      def checkout(branch, opts = {})
+        @git_adapter.checkout(branch, opts)
+      end
+
+      def current_branch
+        @git_adapter.current_branch
+      end
+
+      def diff
+        @git_adapter.diff
+      end
+
+      def diff_name_status(branch_or_sha_1 = nil, branch_or_sha_2 = nil, opts = {})
+        @git_adapter.diff_name_status(branch_or_sha_1, branch_or_sha_2, opts)
+      end
+
+      def fetch(remote = 'origin')
+        @git_adapter.fetch(remote)
+      end
+
+      def head_commit_sha
+        @git_adapter.head_commit_sha
+      end
+
+      def is_there_remote?(remote_name)
+        @git_adapter.is_there_remote?(remote_name)
+      end
+
+      def merge(branch_to_merge_from, opts = {})
+        @git_adapter.merge(branch_to_merge_from, opts)
+      end
+
+      def push(remote, branch, opts = {})
+        @git_adapter.push(remote, branch, opts)
+      end
+
+      def push_from_cached_branch(remote, branch, opts = {})
+        @git_adapter.push_from_cached_branch(remote, branch, opts)
+      end
+
+      def pull(remote, branch)
+        @git_adapter.pull(remote, branch)
+      end
+
+      def remotes
+        @git_adapter.remotes
+      end
+
+      def remove_remote(name)
+        @git_adapter.remove_remote(name)
+      end
+
+      def stage_and_commit(commit_msg = nil)
+        @git_adapter.stage_and_commit(commit_msg)
+      end
+
+      def empty_commit(commit_msg = nil)
+        @git_adapter.empty_commit(commit_msg)
+      end
+
+      def commit(commit_msg = "", opts = {})
+        @git_adapter.commit(commit_msg, opts)
+      end
+
+      def reset_soft(sha)
+        @git_adapter.reset_soft(sha)
+      end
+
+      def reset_hard(sha)
+        @git_adapter.reset_hard(sha)
+      end
+
+      def revparse(string)
+        @git_adapter.revparse(string)
+      end
+
+      def rev_list(base_sha)
+        @git_adapter.rev_list(base_sha)
+      end
+
+      def local_ahead(base_sha, remote_sha)
+        @git_adapter.local_ahead(base_sha, remote_sha)
+      end
+
+      def add_all
+        @git_adapter.add_all
+      end
+
+      def all_branches
+        @git_adapter.all_branches
+      end
+
+      private
+
+      def git_adapter_class
+        self.class.git_adapter_class
+      end
+
+      def self.git_adapter_class
+        GitGem
+      end
+    end
+  end
+end