config_kit 0.0.11

data/lib/config_kit/data/loaders/file_loader.rb

@@ -0,0 +1,68 @@
+require 'yaml'
+module ConfigKit::Data
+  class FileLoader < Loader
+    def self.loader; "file"; end
+
+    attr_reader :files
+    def initialize(uri_kls, env, app, branch)
+      @uri_kls, @env, @app, @branch = uri_kls, env, app, branch
+      @path = File.expand_path('.',File.join(retrieve_path(@uri_kls.path),env))
+      super()
+    end
+
+    def run(&block)
+      return run_all unless block
+      run_batch(&block)
+    end
+
+    def run_all
+      files_data = {}
+      @files.each do |f|
+        files_data.merge!(load_one(f))
+      end
+      files_data
+    end
+
+    def run_batch(&block)
+      while !finish?
+        next_batch
+        files_data = {}
+        @current_files.each do |f|
+          files_data.merge!(load_one(f))
+        end
+        block.call(files_data)
+      end
+    end
+
+    def retrieve_files
+      files = if @app == 'all' 
+        Dir["#{@path}/**/*.yml"].select { |f| match_for?(f, @env) }
+      else
+        Dir["#{@path}/**/*.yml"].select {|f| match_for?(f, @app) && match_for?(f, @env)}
+      end
+      raise ConfigKit::Data::Loader::LoaderFailure.new('No data file found.') if files.empty?
+      files
+    end
+private
+
+    def retrieve_path(path)
+      split_path = path[1..-1].split('/')
+      return path[1..-1] if split_path[0] == '.'
+      path
+    end
+
+    def env_app_for(f)
+      File.basename(f).split('.')[0..1]
+    end
+
+    def load_one(f)
+      env, app = env_app_for(f)
+      raise ConfigKit::Data::Loader::LoaderFailure.new("Wrong data file env(#{env}) for loaded #{@env}") unless env == @env
+      {app => YAML.load_file(f)}
+    end
+
+    def match_for?(f,info)
+      File.basename(f).split('.').find {|a| a == info}
+    end
+  end
+end

data/lib/config_kit/data/loaders/git_loader.rb

@@ -0,0 +1,39 @@
+require 'git'
+module ConfigKit::Data
+  class GitLoader < Loader
+    class GitLoaderError < ConfigKit::Error; end
+    def self.loader; "git"; end
+    attr_reader :clone_path
+    def initialize(uri_kls, env, app, version)
+      @uri_kls, @env, @app, @version = uri_kls, env, app, version
+      @clone_path = Dir.mktmpdir
+      @file_path = "file://#{@clone_path}"
+      @file_kls = URI.parse(@file_path)
+      clone
+    end
+
+    def run(&block)
+      ConfigKit.logger.debug "Git is loading env(#{@env}) #{@app} from #{@uri_kls.to_s}"
+      begin
+        FileLoader.new(@file_kls, @env, @app, @version).run(&block)
+      ensure
+        FileUtils.rm_rf @clone_path
+      end
+    end
+
+private
+    def clone
+      ConfigKit.logger.debug "Git is cloning env(#{@env}) #{@app} from #{@uri_kls.to_s}"
+      begin
+        g = Git.clone(@uri_kls.to_s, @clone_path)
+        tags =  g.tags.map { |t| t.name }
+        raise GitLoaderError.new "Version(#{@version}) not found" unless tags.include?(@version)
+        g.checkout(@version)
+      rescue GitLoaderError => e
+        raise GitLoaderError.new e.message
+      rescue Exception => e
+        raise GitLoaderError.new "Unknown error to load (#{@version}) in #{@uri_kls.to_s}"
+      end
+    end
+  end
+end

data/lib/config_kit/deploy_data.rb

@@ -0,0 +1,78 @@
+require 'config_kit/ext/hash'
+module ConfigKit
+  class DeployData < SlashedHash
+    class DeployDataOptsError < ConfigKit::Error; end
+    def initialize(name, version, opts)
+      @name = name 
+      @version = version[0] == 'v' ? version : "v#{version}"
+      @api_version = opts.fetch(:api_version, '1.0')
+      @kind = opts.fetch(:kind, 'config_kit')
+      @extra = opts.fetch(:extra, :no_default)
+      @cs = opts.fetch(:cs, "no_cs") if @extra != :default_only
+      
+      ConfigKit.logger.debug "Deploy in #{@extra} options"
+      check_extra(@extra)
+      @binded_data = binding_data
+      super(@binded_data)
+    end
+
+    def binding_deploy
+      {'deploy' => data}
+    end
+
+    def binding_kind
+      { @kind => binding_deploy}
+    end
+
+    def binding_default
+      { 
+        @kind => {
+          'deploy' => deploy_default_data
+        }
+      }
+    end
+
+    def binding_data
+      if @extra == :no_default
+        binding_kind
+      elsif @extra == :set_default
+        binding_default.deep_merge(binding_kind)
+      elsif @extra == :default_only
+        binding_default
+      end
+    end
+
+    def data
+      { 
+        @name => { 
+          @version => 
+                {
+                  "ts" => ts,
+                  "cs" => @cs
+                }
+                }
+      }
+    end
+
+    def deploy_default_data
+      {
+        @name => {
+          'default' => @version
+        }
+      }
+    end
+
+private
+    def extra_options
+      [:no_default, :set_default, :default_only]
+    end
+
+    def check_extra(extra)
+      raise DeployDataOptsError.new "extra options(#{extra_options.join('|')}) error: #{extra}." unless extra_options.include?(extra)
+    end
+
+    def ts
+      (Time.now.to_f * 1000).to_i
+    end
+  end
+end

data/lib/config_kit/error.rb

@@ -0,0 +1,3 @@
+module ConfigKit
+  class Error < StandardError; end
+end

data/lib/config_kit/ext/hash.rb

@@ -0,0 +1,6 @@
+class ::Hash
+    def deep_merge(second)
+        merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : Array === v1 && Array === v2 ? v1 | v2 : [:undefined, nil, :nil].include?(v2) ? v1 : v2 }
+        self.merge(second.to_h, &merger)
+    end
+end

data/lib/config_kit/ext/slashed_hash.rb

@@ -0,0 +1,45 @@
+module ConfigKit
+  class SlashedHash < ::Hash
+    class SlashedHashTypeError < Exception; end
+    def initialize(hash, keep_nesting=false)
+      self.merge!(hash) if keep_nesting
+
+      self.merge!(dot_flattened(hash))
+      SlashedHash.symbolize(self)
+    end
+
+    def inspect
+      "#<#{self.class.name}:#{object_id} #{super}>"
+    end
+
+    def to_hash
+      {}.replace(self)
+    end
+
+    def self.symbolize(hash)
+      hash.keys.each do |key|
+        hash[key.to_sym] = hash.delete(key)
+      end
+    end
+
+    protected
+    # turns {'a' => {'b' => 'c'}} into {'a.b' => 'c'}
+    def dot_flattened(nested_hash, names=[], result={})
+      nested_hash.each do |key, val|
+        next if val == nil
+        if val.respond_to?(:has_key?)
+          dot_flattened(val, names + [key], result)
+        elsif val.is_a?(Array)
+          result[(names + [key]).join('/')] = (val.map do |e|
+            raise SlashedHashTypeError.new "Not Support #{e.class.name} in array" unless e.respond_to?(:to_s)
+            e.to_s
+          end).join(',')
+        else
+          result[(names + [key]).join('/')] = val
+        end
+      end
+      result
+    end
+
+  end
+end

data/lib/config_kit/idc_data.rb

@@ -0,0 +1,28 @@
+require 'config_kit/ext/slashed_hash'
+
+module ConfigKit
+  class IDCData < SlashedHash
+    def initialize(name, env, opts)
+      @name, @env = name, env
+      @api_version = opts.fetch(:api_version, '1.0')
+      @kind = opts.fetch(:kind, 'config_kit')
+      @bind_data = binding_data
+      super(@bind_data)
+    end
+
+    def data
+      { 
+        'name' => @name,
+        'env' => @env
+      }
+    end
+
+    def binding_idc
+      { 'idc' => data}
+    end
+
+    def binding_data
+      { @kind => binding_idc }
+    end
+  end
+end

data/lib/config_kit/manager.rb

@@ -0,0 +1,104 @@
+require 'optparse'
+require 'config_kit/tool'
+module ConfigKit
+  class Manager
+    class IDCEnvMissing < ConfigKit::Error; end
+    def self.bootstrap(app, from, uri_kls, version, opts={})
+      opts['uri_kls'] = uri_kls
+      opts['version'] = version
+      opts['from'] = from
+      new(app, opts).bootstrap
+    end
+
+    def self.describe(app, version, opts={})
+      opts['version'] = version
+      new(app, opts).describe
+    end
+
+    def self.rollback(app, version, opts={})
+      opts['version'] = version
+      new(app, opts).rollback
+    end
+
+    def self.deploy(app, version, opts={})
+      opts['version'] = version
+      new(app, opts).deploy
+    end
+
+    def self.init(name, env='int0', opts={})
+      app = 'idc'
+      opts['name'] = name
+      opts['env'] = env
+      opts['skip_env_check'] = true
+      new(app,opts).init
+    end
+
+    def self.get(app,opts={})
+      new(app, opts).get
+    end
+
+    def initialize(app, opts)
+      @app = app
+      @opts = opts
+      @tool = ConfigKit::Tool.new
+      unless opts['skip_env_check'] == true
+        env = @tool.get_idc_env 
+        raise ConfigKit::Manager::IDCEnvMissing.new 'IDC environment missing, pls init it first!' if env.nil?
+        @opts['env'] = env
+      end
+      
+    end
+
+    def init
+      name = @opts.delete('name')
+      env = @opts.delete('env')
+      @tool.init_txn
+      @tool.idc_init_txn(name, env, @opts)
+      @tool.perform_txn
+    end
+
+    def describe
+      version = @opts['version']
+      @tool.describe(@app, version)
+    end
+
+    def bootstrap
+      create
+      {app: @app, version: @opts['version']}
+    end
+
+    def create(extra=:no_default)
+      ConfigKit::Data::Loader.load(@app, @opts['from'], @opts['uri_kls'], @opts['env'], @opts['version']) do |data|
+        @tool.init_txn
+        data.each_pair do |k,v|
+          @tool.bootstrap_txn(v, k)
+          version, cs = get_deploy_info(@tool.config_data)
+          @tool.deploy_txn(k, version, extra,cs)
+        end
+        @tool.perform_txn
+      end
+    end
+
+    def get_deploy_info(config_data)
+      return [] unless config_data.kind_of?(ConfigKit::ConfigData)
+      [config_data.data_version, config_data.data_cs]
+    end
+
+    def get
+      data = @tool.get(@app)[@app]
+      data || {}
+    end
+
+    def deploy
+      change_default
+    end
+
+    def rollback
+      change_default
+    end
+
+    def change_default
+      @tool.deploy_txn(@app, @opts['version'], :default_only)
+    end
+  end
+end

data/lib/config_kit/tool.rb

@@ -0,0 +1,168 @@
+require 'config_kit/config_data'
+require 'config_kit/deploy_data'
+require 'config_kit/idc_data'
+require 'config_kit/ext/hash'
+module ConfigKit
+  class Tool
+    class ConfigMetadataError < ConfigKit::Error; end
+    class MissingDefaultVersionError < ConfigKit::Error;end
+    attr_reader :kind, :namespace, :api_version, :config_data, :deploy_data, :idc_data
+    def initialize(opts={}, api_version=ConfigKit.config.api_version, kind=ConfigKit.config.kind)
+      @kind = kind
+      @api_version = api_version
+      @url = opts.fetch(:url, ConfigKit.config.url)
+      @acl_token = opts.fetch(:acl_token, ConfigKit.config.acl_token)
+      @client = ConfigKit::Client.new(@url, @acl_token)
+    end
+
+    def bootstrap_config(data, name)
+      ConfigKit::ConfigData.new(data, name, api_version, kind)
+    end
+
+    def bootstrap_txn(data,name)
+      @config_data = bootstrap_config(data, name)
+      @client.create_txn(@config_data)  
+    end
+
+    def deploy_config(name, version,extra,cs)
+      ConfigKit::DeployData.new(name, version, extra: extra, cs: cs)
+    end
+
+    def deploy_txn(app,version,extra,cs=nil)
+      if extra != :no_default && check_version_for?(app,version)
+        ConfigKit.logger.debug "Missing version(#{version}) for app(#{app}) to set default"
+        raise MissingDefaultVersionError.new "Missing default version for app(#{app}), pls set default version first.\n"
+      end
+      ConfigKit.logger.debug "compose deploy data for app(#{app}:#{version})"
+      @deploy_data = deploy_config(app,version,extra, cs)
+      @client.create_txn(@deploy_data)
+      @client.perform_txn
+      @deploy_data
+    end
+
+    def check_version_for?(app, version)
+      path = path_for(app, version)
+      @client.read(path).nil?
+    end
+
+    def idc_config(name, env, opts)
+      ConfigKit::IDCData.new(name, env, opts)
+    end
+
+    def idc_init_txn(name, env, opts={})
+      @idc_data = idc_config(name, env, opts)
+      @client.create_txn(@idc_data)
+    end
+
+    def get_idc
+      data =  @client.read(idc_path)
+      return {'idc' => 'N/A'} if data.nil?
+      data['config_kit']['idc']
+    end
+
+    def get_idc_env
+      data =  @client.read(idc_path)
+      return nil if data.nil?
+      data['config_kit']['idc']['env'] 
+    end
+
+    def describe(app, version)
+      @content = {}
+      data = @client.read(idc_path)
+      return @content if data.nil?
+      @content['idc'] = data['config_kit']['idc']
+      data = @client.read(deploy_path)
+      return @content if data.nil?
+      @content['deploy'] = data['config_kit']['deploy']
+      unless app == 'idc'
+        if version.nil?
+          data = @client.read(path_for(app))
+          return @content if data.nil?
+          @content[app] = data['config_kit'][app]
+        else
+          data = @client.read(path_for(app))
+          return @content if data.nil?
+          @content[app] = data['config_kit'][app][version]
+        end
+      end
+      @content
+    end
+
+    def delete_txn(data, name, version=nil)
+      config = bootstrap_config(data)
+    end
+
+    def get(app)
+      return get_all if app == 'all'
+      get_one(app)
+    end
+
+    def get_all()
+      config_data = {}
+      all_apps.each do |app|
+        config_data = config_data.deep_merge(get_one(app))
+      end
+      config_data
+    end
+
+    def get_one(app)
+      context = {}
+      app_data = get_app(app)
+      context[app] = app_data
+      context[app]['idc'] = get_idc
+      unless app == 'infra'
+        infra_data = get_app('infra')['service'] 
+        context[app]['service'] = infra_data
+      end
+      context
+    end
+
+    def get_app(app)
+      version = version_for(app)
+      if version.nil?
+        ConfigKit.logger.debug "Missing default version for app(#{app})"
+        raise MissingDefaultVersionError.new "Missing default version for app(#{app}), pls set default version first.\n"
+      end
+      app_path = path_for(app, version)
+      pp app_path
+      pp @client.read(app_path)
+      app_version = @client.read(app_path)[kind]
+      app_data = {}
+      app_data[app] = app_version[app][version]
+    end
+
+    def all_apps
+      apps = @client.read(deploy_path)[kind]['deploy'].keys
+      apps
+    end
+
+    def version_for(app)
+      @client.read(deploy_app_path(app),false)
+    end
+
+    def init_txn
+      @client.init_txn
+    end
+
+    def perform_txn
+      @client.perform_txn
+    end
+
+    def deploy_app_path(app)
+      "#{deploy_path}/#{app}/default"
+    end
+
+    def deploy_path
+      "/#{kind}/deploy"
+    end
+
+    def path_for(app, version=nil)
+      return "/#{kind}/#{app}" if version.nil?
+      "/#{kind}/#{app}/#{version}" 
+    end
+
+    def idc_path
+      "/#{kind}/idc"
+    end
+  end
+end