dtk-common 0.5.11

data/README.md

@@ -0,0 +1,36 @@
+dtk-common
+==========
+
+DTK Common
+
+
+GITOLITE
+=========
+
+Manager takes responsibility of handling all gitolite methods (or at least most of them). Reason is simple, gitolite commit / push are expensive operations and we want to mitigate that fact by using manager, and making sure that all our changes are under one commit / push.
+
+* Example: Adding user/user group/all to repo configuration
+
+    manager = Gitolite::Manager.new('/home/git/gitolite-admin')
+
+    repo_conf = manager.open_repo('r8--cm--java')
+    repo_conf.add_username_with_rights(
+      'dtk-instance-dtk9', 
+      'RW+'
+    )
+
+    repo_conf.add_user_group_with_rights(
+      'tenants', 
+      'R'
+    )
+
+    repo_conf.add_all_with_rights(
+      gitolite_friendly('RW')
+    )
+
+    manager.push()
+
+
+License
+----------------------
+DTK Common is released under the GPLv3 license. Please see LICENSE for more details.

data/dtk-common.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/dtk-common/version', __FILE__)
+ 
+Gem::Specification.new do |gem|
+  gem.authors       = ["Rich PELAVIN"]
+  gem.email         = ["rich@reactor8.com"]
+  gem.description   = %q{Dtk common is needed to use dtk-client gem, provides common libraries for running DTK CLI.}
+  gem.summary       = %q{Common libraries used for DTK CLI client.}
+  gem.homepage      = "https://github.com/rich-reactor8/dtk-common"
+  gem.licenses      = ["GPL-3.0"]
+
+  gem.files         = `git ls-files`.split($\)
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "dtk-common"
+  gem.require_paths = ["lib"]
+  gem.version       = "#{DtkCommon::VERSION}.#{ARGV[3]}".chomp(".")
+
+  gem.add_dependency 'rugged','~> 0.17.0.b7'
+  gem.add_dependency 'dtk-common-core','~> 0.5.10'
+  gem.add_dependency 'colorize','~> 0.5.8'
+  # gem.add_dependency 'sequel','~> 3.40.0'
+  # gem.add_dependency 'rdoc','~> 3.12'
+end

data/lib/common_patch.rb

@@ -0,0 +1,16 @@
+#TODO: just putting in hooks for errors and logs
+#need to figure out how to hook calling library's errors
+
+module DtkCommon
+  class Error < NameError
+    def initialize(msg,name=nil)
+      super(msg,name)
+      end
+  end
+  class ErrorUsage < Error
+  end
+  Log = ::DTK::Log
+  module Aux
+    extend ::DTK::Common::AuxMixin
+  end
+end

data/lib/dsl/directory_parser/git.rb

@@ -0,0 +1,39 @@
+require File.expand_path('../../git_repo.rb',File.dirname(__FILE__))
+module DtkCommon; module DSL
+  class DirectoryParser
+    class Git < self
+#      def initialize(repo_path,directory_type,branch='master')
+      def initialize(directory_type,repo_path,branch='master')
+        super(directory_type)
+ #       puts repo_path
+        @repo_path = repo_path
+        @repo_branch = GitRepo::Branch.new(repo_path,branch)
+      end
+
+      def self.implements_method?(method_name)
+        if DirectoryParserMethods.include?(method_name)
+          case method_name
+           when :parse_directory
+            GitRepo::Branch.implements_method?(:get_file_content)
+           else
+            true
+          end
+        end
+      end
+
+     private
+      def all_files_from_root()
+        # TODO: Watch version here
+#        output = `git --git-dir=#{@repo_path} ls-tree --full-tree -r HEAD`
+        @repo_branch.list_files()
+      end
+      def get_content(file_path)
+#        output = `git --git-dir=#{@repo_path} show  HEAD:#{file_path}`
+#        output = '{}' if output.empty?
+#        output
+        #If file does not exsist will return nil
+        @repo_branch.get_file_content(file_path)
+      end
+    end
+  end
+end; end

data/lib/dsl/directory_parser/linux.rb

@@ -0,0 +1,21 @@
+module DtkCommon; module DSL
+  class DirectoryParser
+    class Linux < self
+      def initialize(directory_type,directory_root)
+        super(directory_type)
+        @directory_root = directory_root
+      end
+     private
+      def all_files_from_root()
+        Dir.chdir(@directory_root) do
+          Dir["**/*"]
+        end
+      end
+
+      def get_content(rel_file_path)
+        file_path = "#{@directory_root}/#{rel_file_path}"
+        File.open(file_path).read()
+      end
+    end
+  end
+end; end

data/lib/dsl/directory_parser.rb

@@ -0,0 +1,77 @@
+module DtkCommon
+  module DSL
+    class DirectoryParser
+      require File.expand_path("directory_parser/linux",File.dirname(__FILE__))
+      require File.expand_path("directory_parser/git",File.dirname(__FILE__))
+
+      def initialize(directory_type)
+        unless @file_info = file_info(directory_type)
+          raise Error.new("Illegal directory type (#{directory_type})")
+        end
+        @directory_type = directory_type
+      end
+
+      DirectoryParserMethods = [:parse_directory]
+      def self.implements_method?(method_name)
+        DirectoryParserMethods.include?(method_name)
+      end
+
+      #if file_type is given returns DtkCommon::DSL::FileParser::OutputArray
+      #otherwise returns hash at top level taht is indexed by file types found
+      def parse_directory(file_type=nil,opts={})
+        pruned_file_info = 
+          if file_type
+            matches = @file_info.select{|r|r[:file_type] == file_type}
+            if matches.empty?
+              raise Error.new("Illegal file type (#{file_type}) for directory_type (#{directory_type})")
+            end
+            matches
+          else
+            @file_info
+          end
+        #instantiate any rel_path_pattern
+        pruned_file_instances  = instantiate_rel_path_patterns(pruned_file_info)
+        ret = Hash.new
+        pruned_file_instances.each do |r|
+          file_content = get_content(r[:rel_path])
+          opts[:file_path] = r[:rel_path]
+          new_parsed = FileParser.parse_content(r[:file_type],file_content,opts)
+          ret[file_type] = (ret[file_type] ? ret[file_type] + new_parsed : new_parsed)
+        end
+        file_type.nil? ? ret : ret[file_type]
+      end
+
+     private
+      def file_info(directory_type)
+        DirectoryTypeFiles[directory_type]
+      end
+      def instantiate_rel_path_patterns(rel_file_info)
+        ret = Array.new
+        all_files_from_root = nil
+        rel_file_info.each do |r|
+          if rel_path = r[:rel_path]
+            ret << r
+          else
+            rel_path_pattern = r[:rel_path_pattern]
+            
+            (all_files_from_root ||= all_files_from_root()).each do |f|
+              if f =~ rel_path_pattern
+                file_key = $1
+                ret << {:rel_path => f, :file_type => r[:file_type], :key => file_key}
+              end
+            end
+          end
+        end
+        ret
+      end
+      #TODO: may put version info here too
+      DirectoryTypeFiles = {
+        :service_module => 
+        [
+         {:rel_path => "global_module_refs.json", :file_type => :component_module_refs},
+         {:rel_path_pattern => /^assemblies\/([^\/]+)\/assembly\.yaml$/, :file_type => :assembly}
+        ]
+      }
+    end
+  end
+end

data/lib/dsl/file_parser/file_types/assembly/v2/assembly.rb

@@ -0,0 +1,53 @@
+module DtkCommon; module DSL; class FileParser
+  class Assembly < self
+    class OutputArray < FileParser::OutputArray
+      def self.keys_for_row()
+        [:assembly_name,:components]
+      end
+    end
+
+    class V2 < self
+      def parse_hash_content(input_hash)
+        ret = OutputArray.new
+        assembly_hash = OutputHash.new(
+          :assembly_name => input_hash[:name],
+          :components => Component.parse_hash_content(input_hash[:assembly]||{})                                       
+        )
+        ret << assembly_hash
+        ret
+      end
+
+      class Component
+        class OutputArray < FileParser::OutputArray
+          def self.keys_for_row()
+            [:component_name,:module_name,:node_name]
+          end
+        end
+
+        def self.parse_hash_content(input_hash)
+          ret = OutputArray.new
+          (input_hash[:nodes]||{}).each_pair do |node_name,node_info|
+            (node_info[:components]||{}).each do |component|
+              mod_component_name = (component.kind_of?(Hash) ? component.keys.first : component)
+              module_name,component_name = ret_module_and_component_names(mod_component_name)
+              ret << OutputHash.new(:component_name => component_name,:module_name => module_name,:node_name => node_name)
+            end
+          end
+          ret
+        end
+       private
+
+        #returns [module_name,component_name]
+        def self.ret_module_and_component_names(mod_component_name)
+          if mod_component_name =~ /(^[^:]+)::([^:]+$)/
+            [$1,$2]
+          else
+            [mod_component_name,mod_component_name]
+          end
+        end
+      end
+
+    end
+  end
+end; end; end
+

data/lib/dsl/file_parser/file_types/component_module_refs/v1/component_module_refs.rb

@@ -0,0 +1,57 @@
+module DtkCommon; module DSL; class FileParser
+  class ComponentModuleRefs < self
+    class V1 < self
+      def parse_hash_content(input_hash)
+        ret = OutputArray.new
+        component_modules = input_hash[:component_modules]
+        if component_modules.empty?
+          return ret
+        end
+
+        component_modules.each do |component_module,v|
+          new_el = OutputHash.new(:component_module => component_module)
+          parse_error = true
+          if v.kind_of?(InputHash) and v.only_has_keys?(:version,:remote_namespace,:namespace) and not v.empty?()
+            parse_error = false
+            new_el.merge_non_empty!(:version_info => v[:version], :remote_namespace => v[:remote_namespace]||v[:namespace])
+          elsif v.kind_of?(String)
+            parse_error = false
+            new_el.merge_non_empty!(:version_info => v)
+          elsif v.nil?
+            parse_error = false
+          end
+          if parse_error
+            err_msg = (parse_error.kind_of?(String) ? parse_error : "Ill-formed term (#{v.inspect})")
+            raise ErrorUsage::DTKParse.new(err_msg)
+          else
+            ret << new_el
+          end
+        end
+        ret
+      end
+
+      def generate_hash(output_array)
+        component_modules = output_array.inject(Hash.new) do |h,r|
+          unless cmp_module = r[:component_module]
+            raise Error.new("Missing field (:component_module)")
+          end
+          h.merge(cmp_module => Aux.hash_subset(r,OutputArrayToParseHashCols,:no_non_nil => true))
+        end
+        {:component_modules => component_modules} 
+      end
+      
+      OutputArrayToParseHashCols = [{:version_info => :version},:remote_namespace]
+
+    end
+
+    class OutputArray < FileParser::OutputArray
+      def self.keys_for_row()
+        [:component_module,:version_info,:remote_namespace]
+      end
+      def self.has_required_keys?(hash_el)
+        !hash_el[:component_module].nil?
+      end
+    end
+  end
+end; end; end
+

data/lib/dsl/file_parser.rb

@@ -0,0 +1,228 @@
+require 'singleton'
+require 'json'
+require 'yaml'
+
+module DtkCommon
+  module DSL             
+    class Loader
+      include Singleton
+      def self.file_parser(file_type,version=nil)
+        instance.file_parser(file_type,version)
+      end
+
+      def file_parser(file_type,version=nil)
+        ret = (@loaded_types[file_type]||{})[version]
+        return ret if ret
+        unless FileTypes.include?(file_type)
+          raise Error.new("Illegal file type (#{file_type})")
+        end
+
+        #load base if no versions loaded already
+        base_path = "#{BaseDirForFileTypes}/#{file_type}"
+        #if (@loaded_types[file_type]||{}).empty?
+        #   require File.expand_path(base_path,File.dirname(__FILE__))
+        #end
+
+        version ||= default_version(file_type)
+        path = "#{base_path}/v#{version.to_s}/#{file_type}"
+        require File.expand_path(path, File.dirname(__FILE__))
+
+        base_class = FileParser.const_get(Aux.snake_to_camel_case(file_type.to_s))
+        ret_class = base_class.const_get("V#{version.to_s}")
+        input_hash_class = ret_class.const_get "InputHash"
+        ret = ret_class.new(input_hash_class)
+        (@loaded_types[file_type] ||= Hash.new)[version] = ret
+        ret
+      end
+      BaseDirForFileTypes = "file_parser/file_types"
+     private
+      def initialize()
+        @loaded_types = Hash.new
+      end
+
+      def default_version(file_type)
+        FileTypeVesisonDefaults[file_type] || 1
+      end
+      FileTypes = 
+        [
+         :component_module_refs,
+         :assembly
+        ]
+      FileTypeVesisonDefaults = {
+        :component_module_refs => 1,
+        :assembly => 2
+      }
+    end
+    class FileParser
+      def initialize(input_hash_class)
+        @input_hash_class = input_hash_class
+      end
+
+      def self.implements_method?(method_name)
+        [:parse_content,:generate_hash].include?(method_name)
+      end
+
+      def self.parse_content(file_type,file_content,opts={})
+        ret = OutputArray.new
+        # if there is no content (nil) return empty array as if content was empty
+        return ret unless file_content
+        file_parser = Loader.file_parser(file_type,opts[:version])
+        # raw_hash_content = convert_json_content_to_hash(file_content,opts)
+        #TODO: for Rich
+        # we need to implement dsl v3 parser, this is just temp fix for autoimport
+        # just changed parser to yaml instead of json because new modules/services
+        # are in yaml format now 
+        raw_hash_content = convert_yaml_content_to_hash(file_content,opts)
+        
+        return raw_hash_content if raw_hash_content.is_a?(ErrorUsage::DSLParsing::JSONParsing)
+
+        file_parser.parse_hash_content_aux(raw_hash_content)
+      end
+
+      def parse_hash_content_aux(raw_hash)
+        parse_hash_content(input_form(raw_hash))
+      end
+
+      def self.generate_hash(file_type,output_array,opts={})
+        file_parser = Loader.file_parser(file_type,opts[:version])
+        file_parser.generate_hash(output_array)
+      end
+
+      class OutputArray < Array
+        def <<(hash_el)
+          bad_keys = hash_el.keys - self.class.keys_for_row()
+          unless bad_keys.empty?
+            raise Error.new("Illegal keys being inserted in OutputArray (#{bad_keys.join(',')})")
+          end
+          super
+        end
+
+        def +(output_obj)
+          if output_obj.kind_of?(OutputArray)
+            super
+          elsif output_obj.kind_of?(OutputHash)
+            super(OutputArray.new(OutputHash))
+          elsif output_obj.nil?
+            self
+          else
+            raise Error.new("Unexpected object type (#{output_obj.class})")
+          end
+        end
+
+        #can be overwritten
+        def self.has_required_keys?(hash_el)
+          (keys_for_row() - hash_el.keys?).nil?
+        end
+
+      end
+      class OutputHash < ::DTK::Common::SimpleHashObject
+        def merge_non_empty!(hash)
+          hash.each{|k,v| merge!(k => v) unless v.nil? or v.empty?}
+          self
+        end
+
+        def +(output_obj)
+          if output_obj.kind_of?(OutputArray)
+            OutputArray.new(self) + output_obj
+          elsif output_obj.kind_of?(OutputHash)
+            merge(output_obj)
+          elsif output_obj.nil?
+            self
+          else
+            raise Error.new("Unexpected object type (#{output_obj.class})")
+          end
+        end
+      end
+
+      class InputHash < Hash
+        #to provide autovification and use of symbol indexes
+        def initialize(hash=nil)
+          super()
+          return unless hash
+          replace_el = hash.inject(Hash.new) do |h,(k,v)|
+            processed_v = (v.kind_of?(Hash) ? self.class.new(v) : v)
+            h.merge(k =>  processed_v)
+          end
+          replace(replace_el)
+        end
+
+        def [](index)
+          val = super(internal_key_form(index)) || {}
+          (val.kind_of?(Hash) ? self.class.new(val) : val)
+        end
+        def only_has_keys?(*only_has_keys)
+          (keys() - only_has_keys.map{|k|internal_key_form(k)}).empty?
+        end
+       private
+        def internal_key_form(key)
+          key.to_s
+        end
+      end
+
+     private
+      def input_form(raw_hash)
+        @input_hash_class.new(raw_hash)
+      end
+
+      def self.convert_yaml_content_to_hash(content, opts={})
+        ret = Hash.new
+        return ret if content.empty?
+
+        begin 
+          YAML.load(content)
+        rescue Exception => e
+          return ErrorUsage::DSLParsing::JSONParsing.new("YAML parsing error #{e.to_s} in file", opts[:file_path]) if opts[:do_not_raise]
+          raise ErrorUsage::DSLParsing::JSONParsing.new("YAML parsing error #{e.to_s} in file", opts[:file_path])
+        end
+      end
+
+      def self.convert_json_content_to_hash(json_file_content, opts={})
+        ret = Hash.new
+        if json_file_content.empty?
+          return ret
+        end
+
+        begin 
+          ::JSON.parse(json_file_content)
+        rescue ::JSON::ParserError => e
+          # raise ErrorUsage::JSONParse.new(e.to_s)
+          return ErrorUsage::DSLParsing::JSONParsing.new("JSON parsing error #{e.to_s} in file", opts[:file_path]) if opts[:do_not_raise]
+          raise ErrorUsage::DSLParsing::JSONParsing.new("JSON parsing error #{e.to_s} in file", opts[:file_path])
+        end
+      end
+
+    end
+
+    class ErrorUsage < Error
+      class DSLParsing < self
+        def initialize(base_error_msg,file_path_or_opts=nil)
+          file_path,opts = file_path_and_opts(file_path_or_opts)
+          super(local_method_err_msg(base_error_msg,file_path),opts)
+        end
+        private
+        def local_method_err_msg(base_error_msg,file_path=nil)
+          file_ref = (file_path && " (in file: #{file_path})")
+          "#{base_error_msg}#{file_ref}"
+        end
+
+        def file_path_and_opts(file_path_or_opts)
+          file_path = nil
+          opts = Hash.new
+          if file_path_or_opts.kind_of?(Hash)
+            opts = file_path_or_opts
+          else
+            file_path = file_path_or_opts
+          end
+          [file_path,opts]
+        end
+
+        class JSONParsing < self
+        end
+      end
+
+      #when error is dtk content
+      class DTKParse < self
+      end
+    end
+  end
+end

data/lib/dsl.rb

@@ -0,0 +1,23 @@
+
+module DtkCommon
+  module DSL    
+
+    #TODO: just putting in hooks for errors and logs
+    #need to figure out how to hook calling library's errors
+    Error = ::DTK::Error
+    ErrorUsage = ::DTK::ErrorUsage
+
+    Log = ::DTK::Log
+    SimpleHashObject = ::DTK::Common::SimpleHashObject
+    module Aux
+      extend ::DTK::Common::AuxMixin
+    end
+
+    require File.expand_path('dsl/directory_parser', File.dirname(__FILE__))
+    require File.expand_path('dsl/file_parser', File.dirname(__FILE__))
+  end
+end
+
+
+
+

data/lib/dtk-common/version.rb

@@ -0,0 +1,3 @@
+module DtkCommon
+  VERSION = "0.5.11"
+end

data/lib/dtk_common.rb

@@ -0,0 +1,36 @@
+module DTK
+  module Common
+
+    # we are refering to dtk-common-repo folder here
+    POSSIBLE_COMMON_CORE_FOLDERS = ['dtk-common-repo','dtk-common-core']
+
+    require File.expand_path('require_first.rb', File.dirname(__FILE__))
+
+    # this gem needs dtk-common-repo to work we load it
+    unless is_gem_installed?('dtk-common-core')
+      dtk_common_core_folder = POSSIBLE_COMMON_CORE_FOLDERS.find do |folder|
+        path = File.join(File.dirname(__FILE__),'..','..',folder)
+        File.directory?(path)
+      end
+
+      if dtk_common_core_folder
+        require File.expand_path("../../#{dtk_common_core_folder}/lib/dtk_common_core.rb", File.dirname(__FILE__))
+      else
+        raise "Not able to find 'dtk-common-core' gem!"
+      end
+    else
+      # gem installed load from here
+      require 'dtk_common_core'
+    end
+
+
+    # we use sorting to establish deterministic behavior accross systems
+    # Dir.glob will not return list of files in same order each time is run, which led to some bug being present
+    # on some systems and not on the others
+    file_list = Dir.glob("#{File.dirname(__FILE__)}/**/*.rb").sort { |a,b| a <=> b }
+
+    file_list.each do |file|
+      require file unless file.include?('dtk-common.rb') || file.include?('file_access/') || file.include?('require_first.rb') || file.include?('postgres.rb') || file.include?('rugged/')
+    end
+  end
+end

data/lib/dynamic_loader.rb

@@ -0,0 +1,28 @@
+require File.expand_path('common_patch',File.dirname(__FILE__))
+require 'thread'
+
+module DtkCommon
+  class DynmamicLoader
+    def self.load_and_return_adapter_class(adapter_type,adapter_name,opts={})
+      begin
+        caller_dir = caller.first.gsub(/\/[^\/]+$/,"")
+        Lock.synchronize{nested_require_with_caller_dir(caller_dir,"#{adapter_type}/adapters",adapter_name)}
+      rescue LoadError
+        raise Error.new("cannot find #{adapter_type} adapter (#{adapter_name})")
+      end
+      base_class = opts[:base_class] || DtkCommon.const_get(camel_case(adapter_type))
+      base_class.const_get(camel_case(adapter_name))
+    end
+
+   private
+    Lock = Mutex.new
+    def self.nested_require_with_caller_dir(caller_dir,dir,*files_x)
+      files = (files_x.first.kind_of?(Array) ? files_x.first : files_x) 
+      files.each{|f|require File.expand_path("#{dir}/#{f}",caller_dir)}
+    end
+    def self.camel_case(x)
+      Aux.snake_to_camel_case(x.to_s)
+    end
+  end
+end
+

data/lib/git_repo/adapters/rugged/blob.rb

@@ -0,0 +1,14 @@
+module DtkCommon
+  class GitRepo::Adapter::Rugged
+    class Blob < Obj
+      def initialize(repo_branch,rugged_blob)
+        super(repo_branch)
+        @rugged_blob = rugged_blob
+      end
+      
+      def content()
+        lookup(@rugged_blob[:oid]).read_raw.data
+      end
+    end
+  end
+end

data/lib/git_repo/adapters/rugged/commit.rb

@@ -0,0 +1,14 @@
+module DtkCommon
+  class GitRepo::Adapter::Rugged
+    class Commit < Obj
+      def initialize(repo_branch,rugged_commit)
+        super(repo_branch)
+        @rugged_commit = rugged_commit
+      end
+
+      def tree()
+        Tree.new(@repo_branch,@rugged_commit.tree)
+      end
+    end
+  end
+end