dtk-common 0.5.11

data/lib/grit_adapter/file_access.rb

@@ -0,0 +1,296 @@
+require 'colorize'
+
+module DTK; module Common; class GritAdapter
+  class FileAccess < self
+    require File.expand_path('file_access/status', File.dirname(__FILE__))
+    require File.expand_path('file_access/diff', File.dirname(__FILE__))
+    include StatusMixin
+    include DiffMixin
+    def add_file(file_rel_path, content=nil)
+      content ||= String.new
+      file_path = qualified_path(file_rel_path)
+      chdir_and_checkout do
+        File.open(file_path,"w"){|f|f << content}
+        git_command__add(file_path)
+      end
+    end
+
+    def add_file_command(file_rel_path)
+      chdir_and_checkout do
+        file_path = qualified_path(file_rel_path)
+        git_command__add(file_path)
+      end
+    end
+
+    def git_command__add(file_path)
+      #put in -f to avoid error being thrown if try to add an ignored file
+      git_command(:add,file_path,"-f")
+    end
+    private :git_command__add
+
+    def remove_file(file_rel_path)
+      file_path = qualified_path(file_rel_path)
+      chdir_and_checkout do
+        if File.exists?(file_path)
+          git_command(:rm,file_path)
+        end
+      end
+    end
+
+    def pull(remote_branch,local_branch,remote=nil)
+      remote ||= default_remote()
+      chdir do
+        git_command(:pull,remote,"#{remote_branch}:#{local_branch}")
+      end
+    end
+
+    def fetch(remote=nil)
+      remote ||= default_remote()
+      chdir do
+        git_command(:fetch,remote)
+      end
+    end
+
+    #temp branch #TODO: make sure no name conflict
+    TempBranch = 'temp_branch'
+    def merge_theirs(remote_branch_ref)
+      #since there is no 'git merge -s theirs' we need to simulate it
+      chdir do
+        git_command(:checkout,"-b",TempBranch,remote_branch_ref)
+        git_command(:merge,@branch,"-s","ours")
+        git_command(:checkout,@branch)
+        git_command(:reset,"--hard",TempBranch)
+        git_command(:branch,"-D",TempBranch)
+      end
+    end
+
+    def merge(remote_branch_ref)
+      chdir_and_checkout do
+        git_command(:merge,remote_branch_ref)
+      end
+    end
+
+    def changed_files()
+      # NOTE: There is issue with grit and git. Where grit.status will report file changed (modified)
+      # and git status will not. Grit registers changing file time-stamp as change while git doesn't. This would 
+      # not be a problem but `git push` will fail because of this. Following is fix for that.
+      output = git_command_status()
+      grit_files = @grit_repo.status.files.select { |k,v| (v.type =~ /(A|M)/ || v.untracked) }
+      changed_files = grit_files.select do |file|
+        file_name = file.instance_of?(String) ? file : file.first
+        filter_file_through_status_output(file_name,output)
+      end
+
+      # returns array of arrays (first element name of file)
+      changed_files.to_a
+    end
+
+    #TODO: see if we can do away with this
+    def filter_file_through_status_output(file_name,output)
+      #need to search for both filename and whether dircetory added
+       if output.include?(file_name)
+         true
+       else
+         file_parts = file_name.split('/')
+         file_parts.pop
+         while not file_parts.empty? do
+           return true if output.include?(file_parts.join('/')+'/')
+           file_parts.pop
+         end
+         false
+       end
+    end
+    private :filter_file_through_status_output
+
+    def deleted_files()
+      # returns array of arrays (first element name of file)
+      @grit_repo.status.deleted().to_a 
+    end
+
+    def print_status()
+      changes = [@grit_repo.status.changed(), @grit_repo.status.untracked(), @grit_repo.status.deleted()]
+      puts "\nModified files:\n".colorize(:green) unless changes[0].empty?
+      changes[0].each { |item| puts "\t#{item.first}" }
+      puts "\nAdded files:\n".colorize(:yellow) unless changes[1].empty?
+      changes[1].each { |item| puts "\t#{item.first}" }
+      puts "\nDeleted files:\n".colorize(:red) unless changes[2].empty?
+      changes[2].each { |item| puts "\t#{item.first}" }
+      puts ""
+    end
+
+    ##
+    # Checks for changes add/delete/modified
+    #
+    def changed?
+      !(changed_files() + deleted_files).empty?
+    end
+
+    ##
+    # Method will add and remove all files, after commit with given msg
+    #
+    def add_remove_commit_all(commit_msg)
+      chdir do
+        # modified, untracked
+        changed_files().each do |c_file|
+          add_file_command(c_file.first)
+        end
+        # deleted
+        deleted_files().each do |d_file|
+          remove_file(d_file.first)
+        end
+        # commit 
+        commit(commit_msg)
+      end
+    end
+
+    def commit(commit_msg,opts={})
+      cmd_args = [:commit,"-a","-m",commit_msg]
+      author = "#{opts[:author_username]||DefaultAuthor[:username]} <#{opts[:author_email]||DefaultAuthor[:email]}>"
+      cmd_args += ["--author",author]
+      chdir_and_checkout do
+        #note using following because silent failure @grit_repo.commit_all(commit_msg)
+        git_command(*cmd_args)
+      end
+    end
+    DefaultAuthor = {
+      :username => "dtk",
+      :email => "dtk@reactor8.com"
+    }
+
+    # returns :equal, :local_behind, :local_ahead, or :branchpoint
+    # type can be :remote_branch or :local_branch
+    def ret_merge_relationship(type,ref,opts={})
+      if (type == :remote_branch and opts[:fetch_if_needed])
+        #TODO: this fetches all branches on the remote; see if anyway to just fetch a specfic branch
+        #ref will be of form remote_name/branch
+        #TODO: also see if more efficient to use git ls-remote
+        fetch(ref.split("/").first)
+      end
+      other_grit_ref = 
+        case type
+         when :remote_branch
+          @grit_repo.remotes.find{|r|r.name == ref}
+         when :local_branch
+          @grit_repo.heads.find{|r|r.name == ref}
+         else
+          raise Error.new("Illegal type parameter (#{type}) passed to ret_merge_relationship") 
+        end
+
+      local_sha = head_commit_sha()
+      if opts[:ret_commit_shas]
+        opts[:ret_commit_shas][:local_sha] = local_sha
+      end
+
+      unless other_grit_ref
+        if type == :remote_branch
+          return :no_remote_ref
+        end
+        raise Error.new("Cannot find git ref (#{ref})")
+      end
+      other_sha = other_grit_ref.commit.id
+      if opts[:ret_commit_shas]
+        opts[:ret_commit_shas][:other_sha] = other_sha
+      end
+      
+      if other_sha == local_sha 
+        :equal
+      else
+        #shas can be different but  they can have same content so do a git diff
+        unless any_diffs?(local_sha,other_sha)
+          return :equal
+        end
+        #TODO: see if missing or mis-categorizing any condition below
+        if git_command__rev_list_contains?(local_sha,other_sha) then :local_ahead
+        elsif git_command__rev_list_contains?(other_sha,local_sha) then :local_behind
+        else :branchpoint
+        end
+      end
+    end
+
+    def head_commit_sha()
+      head = @grit_repo.heads.find{|r|r.name == @branch}
+      head && head.commit.id
+    end
+    def find_remote_sha(ref)
+      remote = @grit_repo.remotes.find{|r|r.name == ref}
+      remote && remote.commit.id
+    end
+
+    def add_branch?(branch)
+      unless branches().include?(branch)
+        add_branch(branch)
+      end
+    end
+    def add_branch(branch)
+      chdir_and_checkout() do 
+        git_command(:branch,branch)
+      end
+    end
+    def remove_branch?(branch)
+      if branches().include?(branch)
+        remove_branch(branch)
+      end
+    end
+    def remove_branch(branch)
+      checkout_branch = @branch
+      chdir_and_checkout(checkout_branch,:stay_on_checkout_branch => true) do
+        git_command(:branch,"-d",branch)
+      end.first
+    end
+
+   private
+
+    # 
+    # There is issue with Grit 1.8.7 and 1.9.3 version have diffrent returns on changed/deleted files
+    #
+    # 1.8.7 => Returns array of arrays where file name is first elemenet
+    # 1.9.3 => Returns hash where keys are file names
+    #
+    # No need for it now, but when refactoring code use this instead of .to_a fix
+    def grit_compability_transform(grit_files)
+      grit_files.instance_of?(Hash) ? grit_files.keys : grit_files.collect { |element| element.first }
+    end
+
+    def default_remote()
+      "origin"
+    end
+
+    def qualified_path(file_rel_path)
+      "#{@repo_dir}/#{file_rel_path}"
+    end
+
+    def git_command__rev_list_contains?(container_sha,index_sha)
+      rev_list = git_command(:rev_list,container_sha)
+      !rev_list.split("\n").grep(index_sha).empty?()
+    end
+
+    #TODO: would like more efficient way of doing this as opposed to below which first produces object with full diff as opposed to summary
+    def any_diffs?(ref1,ref2)
+      not @grit_repo.diff(ref1,ref2).empty?
+    end
+
+    #TODO: may  determine where --git-dir option makes an actual chdir unnecssary
+    def chdir_and_checkout(branch=nil,opts={},&block)
+      branch ||= @branch
+      chdir do 
+        current_head = @grit_repo.head.name
+        git_command(:checkout,branch) unless current_head == branch
+        return unless block
+        ret = yield
+        unless opts[:stay_on_checkout_branch] or (current_head == branch)
+           git_command(:checkout,current_head)
+        end
+        ret
+      end
+    end
+
+    def chdir(&block)
+      unless File.directory?(@repo_dir)
+        raise Error.new("Trying to chdir to directory (#{@repo_dir}) that does not exist")
+      end
+      Dir.chdir(@repo_dir){yield}
+    end
+
+  end
+end;end;end
+

data/lib/grit_adapter/object_access.rb

@@ -0,0 +1,39 @@
+module DTK::Common; class GritAdapter
+  class ObjectAccess  < self 
+    def initialize(repo_dir,branch=nil)
+      super
+      @grit_index = @grit_repo.index
+    end
+
+    def read_tree()
+      @grit_index.read_tree(@branch)
+    end
+
+    def add_or_replace_file(file_path,content)
+      @grit_index.add(file_path,content)
+    end
+
+    def delete_file(file_path)
+      @grit_index.delete(file_path)
+    end
+
+    def create_branch(new_branch)
+      if branches().include?(new_branch)
+        raise Error.new("Branch (#{new_branch}) already exists")
+      end
+      commit_msg = "adding new branch (#{new_branch})"
+      @grit_index.commit(commit_msg,@grit_repo.commits,nil,nil,new_branch)
+    end
+
+    def commit(commit_msg)
+      @grit_index.commit(commit_msg,@grit_repo.commits,nil,nil,@branch)
+      git_command("write-tree".to_sym)
+    end
+
+    def commit_context(commit_msg,&block)
+      read_tree()
+      yield
+      commit(commit_msg)
+    end
+  end
+end;end

data/lib/grit_adapter.rb

@@ -0,0 +1,212 @@
+#TODO: should I move all methods that user grit.git to file_access?
+require 'grit'
+require 'fileutils'
+require 'thread'
+
+module DTK
+  module Common
+
+    require File.expand_path('require_first',File.dirname(__FILE__))
+    class GritAdapter 
+      require File.expand_path('grit_adapter/file_access', File.dirname(__FILE__))
+      require File.expand_path('grit_adapter/object_access', File.dirname(__FILE__))
+      def initialize(repo_dir,branch=nil,opts={})
+        @repo_dir = repo_dir
+        @branch = branch
+        @grit_repo = nil
+        begin
+          @grit_repo = (opts[:init] ?  init(repo_dir,branch,opts) : create_for_existing_repo(repo_dir,opts))
+          @branch ||= default_branch()
+        rescue ::Grit::NoSuchPathError
+          repo_name = repo_dir.split("/").last.gsub("\.git","")
+          #TODO: change to usage error
+          raise Error.new("repo (#{repo_name}) does not exist")
+        rescue => e
+          raise e
+        end
+      end
+
+      attr_reader :branch,:repo_dir
+
+      def self.clone(target_repo_dir,git_server_url,opts={})
+        if File.directory?(target_repo_dir)
+          if opts[:delete_if_exists]
+            FileUtils.rm_rf target_repo_dir
+          else
+            # raise Error.new("trying to create a repo directory (#{target_repo_dir}) that exists already")
+            raise DTK::Client::DtkError, "trying to create a repo directory (#{target_repo_dir}) that exists already"
+          end
+        end
+        clone_cmd_opts = {:raise => true, :timeout => 60}
+        clone_args = [git_server_url,target_repo_dir]
+        if branch = opts[:branch] 
+          clone_args += ["-b",branch]
+        end
+        ::Grit::Git.new("").clone(clone_cmd_opts,*clone_args)
+        ret = new(*[target_repo_dir,opts[:branch]].compact)
+        #make sure remote branch exists; ::Grit::Git.new("").clone silently uses master if remote branch does not exist
+        if branch = opts[:branch]
+          branches = ret.branches()
+          unless branches.include?(opts[:branch])
+            FileUtils.rm_rf target_repo_dir
+            raise Error.new("Remote branch (#{opts[:branch]}) does not exist")
+          end
+        end
+        ret
+      end
+
+      def branches()
+        @grit_repo.branches.map{|h|h.name}
+      end
+
+      def remotes()
+        @grit_repo.remotes
+      end
+
+      def ls_r(depth=nil,opts={})
+        tree_contents = tree.contents
+        ls_r_aux(depth,tree_contents,opts)
+      end
+
+      def path_exists?(path)
+        not (tree/path).nil?
+      end
+
+      def file_content(path)
+        tree_or_blob = tree/path
+        tree_or_blob && tree_or_blob.kind_of?(::Grit::Blob) && tree_or_blob.data
+      end
+
+      def push(remote_branch_ref=nil)
+        remote_repo,remote_branch = parse_remote_branch_ref(remote_branch_ref)
+        Git_command__push_mutex.synchronize do 
+          git_command(:push,remote_repo||"origin", "#{@branch}:refs/heads/#{remote_branch||@branch}")
+        end
+      end
+      Git_command__push_mutex = Mutex.new
+      #returns [remote_repo,remote_branch]
+      def parse_remote_branch_ref(remote_branch_ref)
+        if remote_branch_ref 
+          split = remote_branch_ref.split("/")
+          case split.size
+            when 1 then [nil,split[0]]
+            when 2 then split
+          end
+        end
+      end
+      private :parse_remote_branch_ref
+
+      def add_remote?(remote_name,remote_url)
+        unless remote_exists?(remote_name)
+          add_remote(remote_name,remote_url)
+        end
+      end
+      def add_remote(remote_name,remote_url)
+        git_command(:remote,"add",remote_name,remote_url)
+      end
+      def add_or_update_remote(remote_name,remote_url)
+        if remote_exists?(remote_name)
+          git_command(:remote,"set-url",remote_name,remote_url)
+        else
+          add_remote(remote_name,remote_url)
+        end
+      end
+
+     private
+
+      def create_for_existing_repo(repo_dir,opts={})
+        unless File.exists?("#{repo_dir}/.git")
+          raise DTK::Client::DtkError, "#{repo_dir} does not contain .git folder." 
+        end
+        ::Grit::Repo.new(repo_dir)
+      end
+
+      def init(repo_dir,branch=nil,opts={})
+        grit_repo = ::Grit::Repo.init(repo_dir)
+        if branch
+          Dir.chdir(repo_dir) do
+            git_command_during_init(grit_repo,"symbolic-ref".to_sym,"HEAD","refs/heads/#{branch}")
+            unless opts[:no_initial_commit]
+              git_command_during_init(grit_repo,:commit,"--allow-empty","-m","initialize")
+            end
+          end
+        end
+        grit_repo
+      end
+
+      def remote_exists?(remote_name)
+        ret_config_keys().include?("remote.#{remote_name}.url")
+      end
+
+      def ret_config_keys()
+        ::Grit::Config.new(@grit_repo).keys
+      end
+      
+      def ret_config_key_value(key)
+        ::Grit::Config.new(@grit_repo).fetch(key)
+      end
+
+      def default_branch()
+        branches = branches()
+        if branches.include?('master')
+          return 'master'
+        elsif branches.size == 1
+          branches.first
+        else
+          raise Error.new("Cannot find a unique default branch")
+        end
+      end
+
+      def tree()
+        @grit_repo.tree(@branch)
+      end
+
+      def ls_r_aux(depth,tree_contents,opts={})
+        ret = Array.new
+        return ret if tree_contents.empty?
+        if depth == 1
+          ret = tree_contents.map do |tc|
+            if opts[:file_only]
+              tc.kind_of?(::Grit::Blob) && tc.name 
+            elsif opts[:directory_only]
+              tc.kind_of?(::Grit::Tree) && tc.name
+            else
+              tc.name
+            end
+          end.compact
+          return ret
+        end
+
+        tree_contents.each do |tc|
+          if tc.kind_of?(::Grit::Blob)
+            unless opts[:directory_only]
+              ret << tc.name
+            end
+          else
+            dir_name = tc.name
+            ret += ls_r_aux(depth && depth-1,tc.contents).map{|r|"#{dir_name}/#{r}"}
+          end
+        end
+        ret
+      end
+
+      def git_command_status()
+        git_command_extra_opts(:status,:chdir => @grit_repo.working_dir)
+      end
+
+      def git_command_during_init(grit_repo,cmd,*args)
+        grit_repo.git.send(cmd, cmd_opts(),*args)
+      end
+      def git_command(cmd,*args)
+        @grit_repo.git.send(cmd, cmd_opts(),*args)
+      end
+      def git_command_extra_opts(cmd,*args)
+        extra_opts = args.pop
+        @grit_repo.git.send(cmd, cmd_opts().merge(extra_opts),*args)
+      end
+      def cmd_opts()
+        {:raise => true, :timeout => 60}
+      end
+    end
+  end
+end

data/lib/module_version.rb

@@ -0,0 +1,12 @@
+module DtkCommon
+  class ModuleVersion
+
+    def self.string_has_version_format?(str)
+      !!(str =~ /\A\d{1,2}\.\d{1,2}\.\d{1,2}\Z/)
+    end
+
+    def self.string_master_or_emtpy?(str)
+      str.empty? || str.casecmp("master") || casecmp("default")
+    end
+  end
+end

data/lib/require_first.rb

@@ -0,0 +1,18 @@
+module DTK
+  module Common
+    def self.r8_require_common(path)
+      require File.expand_path(path, File.dirname(__FILE__))
+    end
+
+    def self.is_gem_installed?(gem_name)
+      begin
+        # if no exception gem is found
+        gem gem_name
+        return true
+      rescue Gem::LoadError
+        return false
+      end
+    end
+  end
+end
+

data/rich_tests/README.md

@@ -0,0 +1 @@
+Make into legitimate ruby unit tests

data/rich_tests/dsl_test1.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+lib_root = '../lib'
+require File.expand_path("#{lib_root}/dsl",File.dirname(__FILE__))
+require 'pp'
+Dir['fixtures/dsl_test1/*/*'].each do |component_module_refs_rel_path|
+  cmr_path = File.expand_path(component_module_refs_rel_path,File.dirname(__FILE__))
+  path_info = component_module_refs_rel_path.split("/")
+  service_mod = path_info[2]
+  puts "processing service module #{service_mod.gsub(/^sm-/,'')}:\n"
+  file_content = File.open(cmr_path).read()
+  pp DtkCommon::DSL::FileParser.parse_content(:component_module_refs,file_content)
+  puts "\n"
+end

data/rich_tests/dsl_test2.rb

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+lib_root = '../lib'
+require File.expand_path("#{lib_root}/dsl",File.dirname(__FILE__))
+require 'pp'
+Dir['fixtures/dsl_test2/*'].each do |service_mod_rel_path|
+  service_mod_path = File.expand_path(service_mod_rel_path,File.dirname(__FILE__))
+  dir_parser = DtkCommon::DSL::DirectoryParser::Linux.new(:service_module,service_mod_path)
+  pp dir_parser.parse_directory(:component_module_refs)
+end

data/rich_tests/dsl_test3.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+lib_root = '../lib'
+require File.expand_path("#{lib_root}/dsl",File.dirname(__FILE__))
+require 'pp'
+
+service_repo_loc = '/home/git/repositories'
+
+if Dir[service_repo_loc]
+  service_directories = Dir.entries(service_repo_loc).select {|entry| entry.to_s.include?('--sm--') }
+  service_directories.each do |service_repo|
+    dir_parser = DtkCommon::DSL::DirectoryParser::Git.new(:service_module, "#{service_repo_loc}/#{service_repo}")
+    pp dir_parser.parse_directory(:component_module_refs)
+  end
+else
+  puts "Not able to find '#{service_repo_loc}' ment to be run on repo-manager"
+end

data/rich_tests/dsl_test4.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+lib_root = '../lib'
+require File.expand_path("#{lib_root}/dsl",File.dirname(__FILE__))
+require 'pp'
+class DtkCommon::DSL::DirectoryParser::Git
+  public :all_files_from_root
+end
+
+Dir.chdir(File.dirname(__FILE__)) do 
+  Dir['fixtures/dsl_test4/*'].each do |service_mod_rel_path|
+    service_mod_path = File.expand_path(service_mod_rel_path,File.dirname(__FILE__))
+    dir_parser = DtkCommon::DSL::DirectoryParser::Git.new(:service_module,service_mod_path)
+
+    puts "-------------------------------------------------"
+    puts "File: #{service_mod_rel_path.split('/').last}"
+    puts "File list"
+    
+    pp dir_parser.all_files_from_root()
+    puts "----\n"
+    puts "Parse service_module component_module_refs"
+    pp dir_parser.parse_directory(:component_module_refs)
+  end
+end
+

data/rich_tests/fixtures/dsl_test1/sm-dtk-bootstrap/global_module_refs.json

@@ -0,0 +1,2 @@
+{
+}

data/rich_tests/fixtures/dsl_test1/sm-dtk-dtk/global_module_refs.json

@@ -0,0 +1,7 @@
+{
+  "component_modules": {
+    "gitolite" : "0.0.1",
+    "dtk_repo_manager": "0.0.1",
+    "thin": "0.0.1"
+  }
+}

data/rich_tests/fixtures/dsl_test1/sm-dtk-test_dtk/global_module_refs.json

@@ -0,0 +1,8 @@
+{
+  "component_modules": {
+    "dtk_postgresql": "0.1.0",
+    "gitolite": "0.1.0",
+    "common_user": "0.1.0",
+    "dtk_server": "0.1.0"
+  }
+}

data/rich_tests/fixtures/dsl_test1/sm-dtk-test_postgres/global_module_refs.json

@@ -0,0 +1,7 @@
+{
+  "component_modules": {
+    "concat": "0.2.0",
+    "dtk_postgresql": "2.2.0",
+    "postgresql": "2.2.0"
+  }
+}

data/rich_tests/fixtures/dsl_test1/sm-dtk-test_service/global_module_refs.json

@@ -0,0 +1,5 @@
+{
+  "component_modules": {
+    "test_mod": "0.1.0"
+  }
+}