config_kit 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +35 -0
- data/Gemfile +11 -0
- data/Gemfile.lock +62 -0
- data/README.md +146 -0
- data/bin/ck +11 -0
- data/cluster/consul.yml +64 -0
- data/config/README.md +134 -0
- data/config/int0/int0.axle.yml +48 -0
- data/config/int0/int0.infra.yml +31 -0
- data/config/stg0/stg0.axle.yml +48 -0
- data/config/stg0/stg0.infra.yml +31 -0
- data/config_kit.gemspec +35 -0
- data/lib/config_kit.rb +20 -0
- data/lib/config_kit/cli/command.rb +77 -0
- data/lib/config_kit/cli/commands/bootstrap.rb +65 -0
- data/lib/config_kit/cli/commands/deploy.rb +48 -0
- data/lib/config_kit/cli/commands/describe.rb +41 -0
- data/lib/config_kit/cli/commands/get.rb +37 -0
- data/lib/config_kit/cli/commands/init.rb +42 -0
- data/lib/config_kit/cli/commands/rollback.rb +40 -0
- data/lib/config_kit/client.rb +203 -0
- data/lib/config_kit/config_data.rb +92 -0
- data/lib/config_kit/configuration.rb +16 -0
- data/lib/config_kit/data/loader.rb +91 -0
- data/lib/config_kit/data/loaders/file_loader.rb +68 -0
- data/lib/config_kit/data/loaders/git_loader.rb +39 -0
- data/lib/config_kit/deploy_data.rb +78 -0
- data/lib/config_kit/error.rb +3 -0
- data/lib/config_kit/ext/hash.rb +6 -0
- data/lib/config_kit/ext/slashed_hash.rb +45 -0
- data/lib/config_kit/idc_data.rb +28 -0
- data/lib/config_kit/manager.rb +104 -0
- data/lib/config_kit/tool.rb +168 -0
- data/lib/config_kit/version.rb +3 -0
- data/scripts/create_config_kit.rb +56 -0
- data/scripts/profile_to_consul.sh +9 -0
- metadata +139 -0
@@ -0,0 +1,37 @@
|
|
1
|
+
module ConfigKit::Cli
|
2
|
+
class Get < Command
|
3
|
+
def self.command; "get"; end
|
4
|
+
|
5
|
+
def initialize(args)
|
6
|
+
@app = 'all'
|
7
|
+
@output = nil
|
8
|
+
super(args)
|
9
|
+
end
|
10
|
+
|
11
|
+
def run
|
12
|
+
begin
|
13
|
+
@output = ConfigKit::Manager.get(@app)
|
14
|
+
pp @output
|
15
|
+
rescue ConfigKit::Cli::Command::CommandFailure
|
16
|
+
raise
|
17
|
+
rescue => e
|
18
|
+
ConfigKit.logger.error "Unexpected error attempting to get config data #{@uri} in env #{@env} for #{@app.nil? ? 'all' : @app}"
|
19
|
+
ConfigKit.logger.debug "#{e}: #{e.backtrace.join("\n ")}"
|
20
|
+
raise ConfigKit::Cli::Command::CommandFailure.new(e.to_s)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
def options
|
26
|
+
OptionParser.new %Q{Usage: #{$0} #{self.class.command} [OPTIONS] ["description"] }, 40 do |opts|
|
27
|
+
opts.separator ''
|
28
|
+
opts.separator 'Specific options:'
|
29
|
+
|
30
|
+
opts.on('-a APP', '--app', 'Specify an app of config to create(default: all to deploy all apps)') do |app|
|
31
|
+
@app = app
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module ConfigKit::Cli
|
2
|
+
class Init < Command
|
3
|
+
def self.command; "init"; end
|
4
|
+
|
5
|
+
def initialize(args)
|
6
|
+
@output
|
7
|
+
super(args)
|
8
|
+
end
|
9
|
+
|
10
|
+
def run
|
11
|
+
begin
|
12
|
+
raise ConfigKit::Cli::Command::CommandFailure.new 'Missing name options' if @name.nil?
|
13
|
+
raise ConfigKit::Cli::Command::CommandFailure.new 'Missing environment options' if @env.nil?
|
14
|
+
@output = ConfigKit::Manager.init(@name, @env)
|
15
|
+
pp @output.to_h.to_json
|
16
|
+
rescue ConfigKit::Cli::Command::CommandFailure
|
17
|
+
raise
|
18
|
+
rescue => e
|
19
|
+
ConfigKit.logger.error "Unexpected error attempting to get config data #{@uri} in env #{@env} for #{@app.nil? ? 'all' : @app}"
|
20
|
+
ConfigKit.logger.debug "#{e}: #{e.backtrace.join("\n ")}"
|
21
|
+
raise ConfigKit::Cli::Command::CommandFailure.new(e.to_s)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
def options
|
27
|
+
OptionParser.new %Q{Usage: #{$0} #{self.class.command} [OPTIONS] ["description"] }, 40 do |opts|
|
28
|
+
opts.separator ''
|
29
|
+
opts.separator 'Specific options:'
|
30
|
+
|
31
|
+
opts.on('-n NAME', '--name', 'Specify a name of IDC to config') do |name|
|
32
|
+
@name = name
|
33
|
+
end
|
34
|
+
|
35
|
+
opts.on('-e ENV', '--env', 'Specify a environment of IDC to config') do |env|
|
36
|
+
@env = env
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module ConfigKit::Cli
|
2
|
+
class Rollback < Command
|
3
|
+
def self.command; "rollback"; end
|
4
|
+
|
5
|
+
def initialize(args)
|
6
|
+
@app = nil
|
7
|
+
super(args)
|
8
|
+
end
|
9
|
+
|
10
|
+
def run
|
11
|
+
begin
|
12
|
+
@output = ConfigKit::Manager.rollback(@app, @version)
|
13
|
+
pp @output.to_h.to_json
|
14
|
+
rescue ConfigKit::Cli::Command::CommandFailure
|
15
|
+
raise
|
16
|
+
rescue => e
|
17
|
+
ConfigKit.logger.error "Unexpected error attempting to get config data #{@uri} in env #{@env} for #{@app.nil? ? 'all' : @app}"
|
18
|
+
ConfigKit.logger.debug "#{e}: #{e.backtrace.join("\n ")}"
|
19
|
+
raise ConfigKit::Cli::Command::CommandFailure.new(e.to_s)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
def options
|
25
|
+
OptionParser.new %Q{Usage: #{$0} #{self.class.command} [OPTIONS] ["description"] }, 40 do |opts|
|
26
|
+
opts.separator ''
|
27
|
+
opts.separator 'Specific options:'
|
28
|
+
|
29
|
+
opts.on('-v VERSION', '--version', 'Specify a existing version to rollout') do |version|
|
30
|
+
@version = version[0] == 'v' ? version : "v#{version}"
|
31
|
+
end
|
32
|
+
|
33
|
+
opts.on('-a APP', '--app', 'Specify an app of config to create(default: all to deploy all apps)') do |app|
|
34
|
+
@app = app
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
@@ -0,0 +1,203 @@
|
|
1
|
+
require 'diplomat'
|
2
|
+
require 'config_kit/error'
|
3
|
+
module ConfigKit
|
4
|
+
class Client
|
5
|
+
class ConfigKitCreateError < ConfigKit::Error; end
|
6
|
+
class ConfigKitUpdateError < ConfigKit::Error; end
|
7
|
+
class ConfigKitTxnError < ConfigKit::Error; end
|
8
|
+
class ConfigKitReadError < ConfigKit::Error; end
|
9
|
+
attr_reader :url, :acl_token, :opts
|
10
|
+
|
11
|
+
class ConsulConnection
|
12
|
+
attr_reader :config, :connection
|
13
|
+
|
14
|
+
extend Forwardable
|
15
|
+
def_delegators :@connection, :put, :get, :delete, :txn
|
16
|
+
def initialize(url, acl_token)
|
17
|
+
setup_consul(url, acl_token)
|
18
|
+
end
|
19
|
+
|
20
|
+
def setup_consul(url, acl_token)
|
21
|
+
@config = Diplomat.configure do |config|
|
22
|
+
config.url = url
|
23
|
+
config.acl_token = acl_token unless acl_token.nil?
|
24
|
+
end
|
25
|
+
@connection = Diplomat
|
26
|
+
end
|
27
|
+
|
28
|
+
def put!(key, value, options = nil)
|
29
|
+
response = put(key, value, options = nil)
|
30
|
+
raise ConfigKitUpdateError, "Config Kit Update key:#{key} error" unless response
|
31
|
+
response
|
32
|
+
end
|
33
|
+
|
34
|
+
def create_txn(data)
|
35
|
+
prepare_txn_data(data,type='create')
|
36
|
+
self
|
37
|
+
end
|
38
|
+
|
39
|
+
def read_txn(data,recurse=false)
|
40
|
+
prepare_txn_data(data,type='read',recurse)
|
41
|
+
self
|
42
|
+
end
|
43
|
+
|
44
|
+
def update_txn(data)
|
45
|
+
prepare_txn_data(data,type='read')
|
46
|
+
self
|
47
|
+
end
|
48
|
+
|
49
|
+
def delete_txn(data,recurse=false)
|
50
|
+
prepare_txn_data(data,type='delete',recurse)
|
51
|
+
self
|
52
|
+
end
|
53
|
+
|
54
|
+
def init_txn
|
55
|
+
reset_txn_data
|
56
|
+
end
|
57
|
+
|
58
|
+
def perform_txn
|
59
|
+
begin
|
60
|
+
txn(@txn_data)
|
61
|
+
rescue => e
|
62
|
+
raise ConfigKitTxnError.new "perform txn error:#{e.message}"
|
63
|
+
ensure
|
64
|
+
reset_txn_data
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def reset_txn_data
|
69
|
+
@txn_data = []
|
70
|
+
end
|
71
|
+
|
72
|
+
def prepare_txn_data(data, type='read', recurse=false)
|
73
|
+
init_txn if @txn_data.nil?
|
74
|
+
verb = determine_verb(type, recurse)
|
75
|
+
data.each_pair do |k,v|
|
76
|
+
kv = {
|
77
|
+
"Verb" => verb,
|
78
|
+
"Key" => k.to_s,
|
79
|
+
"Value" => v.to_s
|
80
|
+
}
|
81
|
+
|
82
|
+
@txn_data << {"KV" => kv}
|
83
|
+
end
|
84
|
+
@txn_data
|
85
|
+
end
|
86
|
+
|
87
|
+
def determine_verb(_type, recurse)
|
88
|
+
verb = nil
|
89
|
+
type = _type.to_s
|
90
|
+
case type
|
91
|
+
when 'read'
|
92
|
+
if recurse
|
93
|
+
verb = 'get-tree'
|
94
|
+
else
|
95
|
+
verb = 'get'
|
96
|
+
end
|
97
|
+
when 'update'
|
98
|
+
verb = 'set'
|
99
|
+
when 'create'
|
100
|
+
verb = 'set'
|
101
|
+
when 'delete'
|
102
|
+
if recurse
|
103
|
+
verb = 'delete-tree'
|
104
|
+
else
|
105
|
+
verb = 'delete'
|
106
|
+
end
|
107
|
+
else
|
108
|
+
verb = nil
|
109
|
+
end
|
110
|
+
verb
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def initialize(url, acl_token)
|
115
|
+
@url = url
|
116
|
+
@acl_token = acl_token
|
117
|
+
@connection = ConsulConnection.new(@url, @acl_token)
|
118
|
+
end
|
119
|
+
|
120
|
+
def create(key,value)
|
121
|
+
response = @connection.put(key, value, cas: 0)
|
122
|
+
raise ConfigKitCreateError, "Config Kit create #{key} error" unless response
|
123
|
+
response
|
124
|
+
end
|
125
|
+
|
126
|
+
def update(key, value)
|
127
|
+
response = @connection.put(key, value)
|
128
|
+
raise ConfigKitUpdateError, "Config Kit update #{key} error" unless response
|
129
|
+
end
|
130
|
+
|
131
|
+
def atom_update(key,value)
|
132
|
+
begin
|
133
|
+
retries ||= 0
|
134
|
+
modify_idx = @connection.get(key, modify_index: true)
|
135
|
+
response = @connection.put!(key, value, cas: modify_idx)
|
136
|
+
rescue ConfigKitUpdateError => e
|
137
|
+
if (retries += 1) < 3
|
138
|
+
#
|
139
|
+
# TODO:
|
140
|
+
# 1. Need to log
|
141
|
+
# 2. Need to delay and retry refactor
|
142
|
+
#
|
143
|
+
sleep(0.5)
|
144
|
+
retry
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
#
|
150
|
+
# TODO: enhancement
|
151
|
+
#
|
152
|
+
def read(key, convert_to_hash=true, recurse=true)
|
153
|
+
begin
|
154
|
+
ConfigKit.logger.debug "getting key: #{key}"
|
155
|
+
response = @connection.get(key, convert_to_hash: convert_to_hash, recurse: recurse)
|
156
|
+
response
|
157
|
+
rescue Diplomat::KeyNotFound => e
|
158
|
+
return nil
|
159
|
+
rescue Faraday::ConnectionFailed => e
|
160
|
+
raise ConfigKitReadError, "config server #{@url} is not avaliable. #{e.message}"
|
161
|
+
rescue Diplomat::UnknownStatus => e
|
162
|
+
raise ConfigKitReadError, "Unknown error #{e.message}."
|
163
|
+
rescue Exception => e
|
164
|
+
ConfigKit.logger.debug e.backtrace.join("\n ")
|
165
|
+
return nil
|
166
|
+
end
|
167
|
+
|
168
|
+
end
|
169
|
+
|
170
|
+
#
|
171
|
+
# TODO: enhancement
|
172
|
+
#
|
173
|
+
|
174
|
+
def delete(key)
|
175
|
+
response = @connection.delete(url, recurse: true)
|
176
|
+
response
|
177
|
+
end
|
178
|
+
|
179
|
+
def create_txn(data)
|
180
|
+
@connection.create_txn(data)
|
181
|
+
end
|
182
|
+
|
183
|
+
def read_txn(data)
|
184
|
+
@connection.read_txn(data)
|
185
|
+
end
|
186
|
+
|
187
|
+
def update_txn(data)
|
188
|
+
@connection.update_txn(data)
|
189
|
+
end
|
190
|
+
|
191
|
+
def delete_txn(data)
|
192
|
+
@connection.delete_txn(data)
|
193
|
+
end
|
194
|
+
|
195
|
+
def init_txn
|
196
|
+
@connection.init_txn
|
197
|
+
end
|
198
|
+
|
199
|
+
def perform_txn
|
200
|
+
@connection.perform_txn
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'config_kit/ext/slashed_hash'
|
2
|
+
|
3
|
+
module ConfigKit
|
4
|
+
class ConfigData < SlashedHash
|
5
|
+
class MetadataError < ConfigKit::Error; end
|
6
|
+
class MetadataMissingRequireFieldError < ConfigKit::Error; end
|
7
|
+
|
8
|
+
class DataError < ConfigKit::Error; end
|
9
|
+
class DataMissingRequireFieldError < ConfigKit::Error; end
|
10
|
+
|
11
|
+
def initialize(config_data, name, api_version='v1.0', kind='config_kit')
|
12
|
+
|
13
|
+
@defined_api_version=api_version
|
14
|
+
@defined_kind = kind
|
15
|
+
|
16
|
+
@metadata_required_fields = ['api_version', 'kind']
|
17
|
+
@data_required_fields = ['version', 'namespace']
|
18
|
+
@config_data = config_data
|
19
|
+
@name = name
|
20
|
+
check_metadata
|
21
|
+
check_data
|
22
|
+
@binded_data = binding_data
|
23
|
+
super(@binded_data)
|
24
|
+
end
|
25
|
+
|
26
|
+
def check_data
|
27
|
+
raise DataError, 'Config Kit Support Hash Data Formate ONLY.' unless data.is_a?(Hash)
|
28
|
+
@data_required_fields.each do |f|
|
29
|
+
raise DataMissingRequireFieldError,"Config Kit Raw data missing require field #{f} " unless data.has_key?(f)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def check_metadata
|
34
|
+
raise MetadataError, 'Config Kit Support Hash Data Formate ONLY.' unless @config_data.is_a?(Hash)
|
35
|
+
@metadata_required_fields.each do |f|
|
36
|
+
raise MetadataMissingRequireFieldError,"Config Kit Meta data missing require field #{f} " unless @config_data.has_key?(f)
|
37
|
+
end
|
38
|
+
raise MetadataError, "Config Kit API Version metadata mis-match #{@config_data['api_version']}." unless support_api?
|
39
|
+
raise MetadataError, "Config Kit Kind metadate mis-match #{@config_data['kind']}." unless same_kind?
|
40
|
+
end
|
41
|
+
|
42
|
+
def binding_version
|
43
|
+
{ data_version => data }
|
44
|
+
end
|
45
|
+
|
46
|
+
def binding_name
|
47
|
+
{ @name => binding_version }
|
48
|
+
end
|
49
|
+
|
50
|
+
def binding_kind
|
51
|
+
{ kind => binding_name }
|
52
|
+
end
|
53
|
+
|
54
|
+
#
|
55
|
+
# /v1/config_kit/axle/v1.0/database => mysql
|
56
|
+
#
|
57
|
+
|
58
|
+
def binding_data
|
59
|
+
binding_kind
|
60
|
+
end
|
61
|
+
|
62
|
+
def same_kind?
|
63
|
+
@defined_kind == kind
|
64
|
+
end
|
65
|
+
|
66
|
+
def support_api?
|
67
|
+
@defined_api_version == api_version
|
68
|
+
end
|
69
|
+
|
70
|
+
def data_version
|
71
|
+
_version = @config_data && @config_data[@name] && @config_data[@name]['version']
|
72
|
+
version = _version[0] == 'v' ? _version : "v#{_version}"
|
73
|
+
version
|
74
|
+
end
|
75
|
+
|
76
|
+
def data_cs
|
77
|
+
nil
|
78
|
+
end
|
79
|
+
|
80
|
+
def data
|
81
|
+
@config_data && @config_data[@name]
|
82
|
+
end
|
83
|
+
|
84
|
+
def api_version
|
85
|
+
@config_data['api_version']
|
86
|
+
end
|
87
|
+
|
88
|
+
def kind
|
89
|
+
@config_data['kind']
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'logger'
|
2
|
+
module ConfigKit
|
3
|
+
class Configuration
|
4
|
+
attr_accessor :url, :acl_token, :options, :api_version, :kind, :debug, :logger
|
5
|
+
def initialize(url, opts={})
|
6
|
+
@url = url
|
7
|
+
@opts = opts
|
8
|
+
@acl_token = opts[:acl_token]
|
9
|
+
@api_version = 'v1.0'
|
10
|
+
@kind = 'config_kit'
|
11
|
+
@logger = @opts.fetch(:logger, ::Logger.new(STDOUT))
|
12
|
+
@debug = @opts[:debug] || false
|
13
|
+
@logger.level = ::Logger::DEBUG if @debug
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'uri'
|
2
|
+
module ConfigKit
|
3
|
+
module Data
|
4
|
+
class Loader
|
5
|
+
|
6
|
+
class LoaderFailure < StandardError
|
7
|
+
attr_reader :options
|
8
|
+
def initialize message, opts=nil
|
9
|
+
super message
|
10
|
+
@options = opts
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.info(message)
|
15
|
+
STDOUT.puts message
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.err(message)
|
19
|
+
STDERR.puts message
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
@loaders = []
|
24
|
+
|
25
|
+
def self.inherited(subclass)
|
26
|
+
@loaders << subclass
|
27
|
+
end
|
28
|
+
|
29
|
+
ldrs = ldrs = File.expand_path(File.join(File.dirname(__FILE__), 'loaders', '*.rb'))
|
30
|
+
Dir[ldrs].each {|loader| require loader}
|
31
|
+
|
32
|
+
def self.load(app, from, uri_kls, env, version, &block)
|
33
|
+
@loader_names = @loaders.map{ |l| l.loader}
|
34
|
+
|
35
|
+
@uri_kls, @env, @app, @version = uri_kls, env, app, version
|
36
|
+
|
37
|
+
loader = from
|
38
|
+
|
39
|
+
|
40
|
+
if loader.nil?
|
41
|
+
ConfigKit.logger.error "from empty source with #{@uri_kls.to_s}"
|
42
|
+
elsif !@loader_names.include?(loader)
|
43
|
+
ConfigKit.logger.error "Unrecognize loader: #{loader} for #{@uri}"
|
44
|
+
else
|
45
|
+
loader_class = @loaders.find{ |c| c.loader == loader}
|
46
|
+
ConfigKit.logger.debug "#{loader_class.loader} is loading env(#{@env}) app(#{app}) data(#{@version}) from #{uri_kls.to_s}"
|
47
|
+
loader_class.new(@uri_kls, @env, @app, @version).run(&block)
|
48
|
+
ConfigKit.logger.debug "#{loader_class.loader} is loaded env(#{@env}) app(#{app}) data(#{@version}) from #{uri_kls.to_s}"
|
49
|
+
end
|
50
|
+
rescue => e
|
51
|
+
raise ConfigKit::Data::Loader::LoaderFailure.new e.message
|
52
|
+
end
|
53
|
+
|
54
|
+
attr_reader :cursor,:batch_size, :files
|
55
|
+
def initialize(batch_size=10)
|
56
|
+
@files = []
|
57
|
+
@batch_size = batch_size
|
58
|
+
@current_files = []
|
59
|
+
@cursor = 0
|
60
|
+
@files = retrieve_files
|
61
|
+
end
|
62
|
+
|
63
|
+
def file_count
|
64
|
+
@files.count
|
65
|
+
end
|
66
|
+
|
67
|
+
def next_batch
|
68
|
+
if finish?
|
69
|
+
@current_files = []
|
70
|
+
@cursor = file_count
|
71
|
+
return @cursor
|
72
|
+
end
|
73
|
+
_next_cursor = next_cursor
|
74
|
+
@current_files = @files[@cursor.._next_cursor - 1]
|
75
|
+
@cursor = _next_cursor
|
76
|
+
end
|
77
|
+
|
78
|
+
def next_cursor
|
79
|
+
@cursor + @batch_size > file_count ? file_count : (@cursor + @batch_size)
|
80
|
+
end
|
81
|
+
|
82
|
+
def finish?
|
83
|
+
@cursor >= file_count
|
84
|
+
end
|
85
|
+
|
86
|
+
def retrieve_files
|
87
|
+
raise NotImplementedError
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|