dop_common 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +23 -0
- data/.rspec +2 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +176 -0
- data/Gemfile +11 -0
- data/LICENSE.txt +177 -0
- data/README.md +48 -0
- data/Rakefile +49 -0
- data/Vagrantfile +25 -0
- data/bin/dop-puppet-autosign +56 -0
- data/doc/examples/example_deploment_plan_v0.0.1.yaml +302 -0
- data/doc/plan_format_v0.0.1.md +919 -0
- data/doc/plan_format_v0.0.2_snippets.md +56 -0
- data/dop_common.gemspec +44 -0
- data/lib/dop_common/affinity_group.rb +57 -0
- data/lib/dop_common/cli/global_options.rb +37 -0
- data/lib/dop_common/cli/log.rb +51 -0
- data/lib/dop_common/cli/node_selection.rb +62 -0
- data/lib/dop_common/command.rb +125 -0
- data/lib/dop_common/config/helper.rb +39 -0
- data/lib/dop_common/config.rb +66 -0
- data/lib/dop_common/configuration.rb +37 -0
- data/lib/dop_common/credential.rb +152 -0
- data/lib/dop_common/data_disk.rb +62 -0
- data/lib/dop_common/dns.rb +55 -0
- data/lib/dop_common/hash_parser.rb +241 -0
- data/lib/dop_common/hooks.rb +81 -0
- data/lib/dop_common/infrastructure.rb +160 -0
- data/lib/dop_common/infrastructure_properties.rb +185 -0
- data/lib/dop_common/interface.rb +113 -0
- data/lib/dop_common/log.rb +78 -0
- data/lib/dop_common/network.rb +85 -0
- data/lib/dop_common/node/config.rb +159 -0
- data/lib/dop_common/node.rb +442 -0
- data/lib/dop_common/node_filter.rb +74 -0
- data/lib/dop_common/plan.rb +188 -0
- data/lib/dop_common/plan_cache.rb +83 -0
- data/lib/dop_common/plan_store.rb +263 -0
- data/lib/dop_common/pre_processor.rb +73 -0
- data/lib/dop_common/run_options.rb +56 -0
- data/lib/dop_common/signal_handler.rb +58 -0
- data/lib/dop_common/state_store.rb +95 -0
- data/lib/dop_common/step.rb +200 -0
- data/lib/dop_common/step_set.rb +41 -0
- data/lib/dop_common/thread_context_logger.rb +77 -0
- data/lib/dop_common/utils.rb +106 -0
- data/lib/dop_common/validator.rb +53 -0
- data/lib/dop_common/version.rb +3 -0
- data/lib/dop_common.rb +32 -0
- data/lib/hiera/backend/dop_backend.rb +94 -0
- data/lib/hiera/dop_logger.rb +20 -0
- data/spec/data/fake_hook_file_invalid +1 -0
- data/spec/data/fake_hook_file_valid +5 -0
- data/spec/data/fake_keyfile +1 -0
- data/spec/dop-puppet-autosign_spec_disable.rb +33 -0
- data/spec/dop_common/affinity_group_spec.rb +41 -0
- data/spec/dop_common/command_spec.rb +83 -0
- data/spec/dop_common/credential_spec.rb +73 -0
- data/spec/dop_common/data_disk_spec.rb +165 -0
- data/spec/dop_common/dns_spec.rb +33 -0
- data/spec/dop_common/hash_parser_spec.rb +181 -0
- data/spec/dop_common/hooks_spec.rb +33 -0
- data/spec/dop_common/infrastructure_properties_spec.rb +224 -0
- data/spec/dop_common/infrastructure_spec.rb +77 -0
- data/spec/dop_common/interface_spec.rb +192 -0
- data/spec/dop_common/network_spec.rb +92 -0
- data/spec/dop_common/node_filter_spec.rb +70 -0
- data/spec/dop_common/node_spec.rb +623 -0
- data/spec/dop_common/plan_cache_spec.rb +46 -0
- data/spec/dop_common/plan_spec.rb +136 -0
- data/spec/dop_common/plan_store_spec.rb +194 -0
- data/spec/dop_common/pre_processor_spec.rb +27 -0
- data/spec/dop_common/run_options_spec.rb +65 -0
- data/spec/dop_common/signal_handler_spec.rb +31 -0
- data/spec/dop_common/step_set_spec.rb +21 -0
- data/spec/dop_common/step_spec.rb +175 -0
- data/spec/dop_common/utils_spec.rb +27 -0
- data/spec/dop_common/validator_spec.rb +47 -0
- data/spec/example_plans_spec.rb +16 -0
- data/spec/fixtures/example_ssh_key +27 -0
- data/spec/fixtures/example_ssh_key.pub +1 -0
- data/spec/fixtures/incl/root_part.yaml +1 -0
- data/spec/fixtures/incl/some_list.yaml +2 -0
- data/spec/fixtures/other_plan_same_nodes.yaml +19 -0
- data/spec/fixtures/simple_include.yaml +6 -0
- data/spec/fixtures/simple_include_with_errors.yaml +4 -0
- data/spec/fixtures/simple_plan.yaml +19 -0
- data/spec/fixtures/simple_plan_invalid.yaml +18 -0
- data/spec/fixtures/simple_plan_modified.yaml +21 -0
- data/spec/spec_helper.rb +106 -0
- metadata +381 -0
@@ -0,0 +1,58 @@
|
|
1
|
+
#
|
2
|
+
# Defered signal handling
|
3
|
+
#
|
4
|
+
# This class will handle the trapping of signals
|
5
|
+
#
|
6
|
+
# This code uses tricks from http://timuruski.net/blog/2014/graceful-shutdown
|
7
|
+
# for setup and teardown of the signal handlers. And the self pipe trick from
|
8
|
+
# http://www.sitepoint.com/the-self-pipe-trick-explained
|
9
|
+
#
|
10
|
+
|
11
|
+
module DopCommon
|
12
|
+
class SignalHandler
|
13
|
+
DEFAULT_SIGNALS = [:INT, :QUIT, :TERM]
|
14
|
+
|
15
|
+
def initialize
|
16
|
+
@signal_queue = []
|
17
|
+
@self_reader, @self_writer = IO.pipe
|
18
|
+
end
|
19
|
+
|
20
|
+
def handle_signals(*signals)
|
21
|
+
signals = DEFAULT_SIGNALS if signals.empty?
|
22
|
+
old_handlers = setup_signal_traps(signals)
|
23
|
+
loop do
|
24
|
+
begin
|
25
|
+
if @signal_queue.any?
|
26
|
+
yield(@signal_queue.shift) if block_given?
|
27
|
+
else
|
28
|
+
IO.select([@self_reader])
|
29
|
+
@self_reader.read_nonblock(1)
|
30
|
+
end
|
31
|
+
rescue
|
32
|
+
break
|
33
|
+
end
|
34
|
+
end
|
35
|
+
teardown_signal_traps(old_handlers)
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def setup_signal_traps(signals)
|
41
|
+
signals.each_with_object({}) do |signal, old_handlers|
|
42
|
+
DopCommon.log.debug("Installing trap for signal #{signal.to_s}")
|
43
|
+
old_handlers[signal] = Signal.trap(signal) do
|
44
|
+
@signal_queue << { signal => Time.now }
|
45
|
+
@self_writer.write_nonblock('.')
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def teardown_signal_traps(old_handlers)
|
51
|
+
old_handlers.each do |signal, old_handler|
|
52
|
+
DopCommon.log.debug("Removing trap for signal #{signal.to_s}")
|
53
|
+
Signal.trap(signal, old_handler)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
#
|
2
|
+
# This is a simple wrapper around YAML::Store to make versioning
|
3
|
+
# and updating easier.
|
4
|
+
#
|
5
|
+
require 'yaml/store'
|
6
|
+
require 'rb-inotify'
|
7
|
+
|
8
|
+
module DopCommon
|
9
|
+
class UnknownVersionError < StandardError
|
10
|
+
end
|
11
|
+
|
12
|
+
class StateStore < YAML::Store
|
13
|
+
|
14
|
+
def initialize(state_file, plan_name, plan_cache)
|
15
|
+
@plan_name = plan_name
|
16
|
+
@plan_cache = plan_cache
|
17
|
+
@state_file = state_file
|
18
|
+
@write_mutex = Mutex.new
|
19
|
+
super(@state_file)
|
20
|
+
end
|
21
|
+
|
22
|
+
# This is a wrapper around transaction to make sure we have a run lock.
|
23
|
+
# This will ensure that only ever one instance can write to this store.
|
24
|
+
def transaction(read_only = false, &block)
|
25
|
+
if read_only
|
26
|
+
super(read_only, &block)
|
27
|
+
else
|
28
|
+
@write_mutex.synchronize do
|
29
|
+
if @plan_cache.run_lock?(@plan_name)
|
30
|
+
# save the version on first write
|
31
|
+
super do
|
32
|
+
self[:version] = latest_version if self[:version].nil?
|
33
|
+
end
|
34
|
+
super(&block)
|
35
|
+
else
|
36
|
+
raise StandardError, "Not possible to write to #{@state_file} because we have no run lock"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def version
|
43
|
+
transaction(true) do
|
44
|
+
self[:version] || :new
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def pending_updates?
|
49
|
+
case version
|
50
|
+
when :new, latest_version then false
|
51
|
+
else true
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
# update the state file. This takes a block which will receive a
|
56
|
+
# hash diff from the state version to the newest plan version.
|
57
|
+
# If the plan is new or already on the latest version the block will
|
58
|
+
# not be executed.
|
59
|
+
#
|
60
|
+
# The block is already inside a transaction. The version will be bumped
|
61
|
+
# to the latest version only if the transaction is successful.
|
62
|
+
def update
|
63
|
+
ver = version
|
64
|
+
return if ver == latest_version
|
65
|
+
return if ver == :new
|
66
|
+
raise UnknownVersionError.new(ver) unless version_exists?(ver)
|
67
|
+
DopCommon.log.info("Updating plan #{@plan_name} from version #{ver} to #{latest_version}")
|
68
|
+
transaction do
|
69
|
+
yield(@plan_cache.get_plan_hash_diff(@plan_name, ver, latest_version))
|
70
|
+
self[:version] = latest_version
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
# This method will take a block which will be executet every time the
|
75
|
+
# state file changes.
|
76
|
+
def on_change
|
77
|
+
notifier = INotify::Notifier.new
|
78
|
+
notifier.watch(@state_file, :modify) do
|
79
|
+
yield
|
80
|
+
end
|
81
|
+
notifier.run
|
82
|
+
end
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
def latest_version
|
87
|
+
@plan_cache.show_versions(@plan_name).last
|
88
|
+
end
|
89
|
+
|
90
|
+
def version_exists?(version)
|
91
|
+
@plan_cache.show_versions(@plan_name).include?(version)
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,200 @@
|
|
1
|
+
#
|
2
|
+
# DOP common step hash parser
|
3
|
+
#
|
4
|
+
|
5
|
+
module DopCommon
|
6
|
+
class Step
|
7
|
+
include Validator
|
8
|
+
include RunOptions
|
9
|
+
|
10
|
+
def initialize(hash)
|
11
|
+
@hash = HashParser.symbolize_keys(hash)
|
12
|
+
HashParser.key_aliases(@hash, :commands, [:command])
|
13
|
+
end
|
14
|
+
|
15
|
+
def name
|
16
|
+
@name ||= @hash[:name] or
|
17
|
+
raise PlanParsingError, "Every step needs to have a 'name' key defined"
|
18
|
+
end
|
19
|
+
|
20
|
+
def validate
|
21
|
+
valitdate_shared_options
|
22
|
+
log_validation_method('name')
|
23
|
+
log_validation_method('nodes_valid?')
|
24
|
+
log_validation_method('exclude_nodes_valid?')
|
25
|
+
log_validation_method('nodes_by_config_valid?')
|
26
|
+
log_validation_method('exclude_nodes_by_config_valid?')
|
27
|
+
log_validation_method('roles_valid?')
|
28
|
+
log_validation_method('exclude_roles_valid?')
|
29
|
+
log_validation_method('commands_valid?')
|
30
|
+
r_name = @name || 'unknown' # name may not be set because of a previous error
|
31
|
+
try_validate_obj("Step #{r_name}: Can't validate the commands part because of a previous error"){commands}
|
32
|
+
end
|
33
|
+
|
34
|
+
def nodes
|
35
|
+
@nodes ||= nodes_valid? ?
|
36
|
+
HashParser.parse_pattern_list(@hash, :nodes) : []
|
37
|
+
end
|
38
|
+
|
39
|
+
def exclude_nodes
|
40
|
+
@exclude_nodes ||= exclude_nodes_valid? ?
|
41
|
+
HashParser.parse_pattern_list(@hash, :exclude_nodes) : []
|
42
|
+
end
|
43
|
+
|
44
|
+
def nodes_by_config
|
45
|
+
@nodes_by_config ||= nodes_by_config_valid? ?
|
46
|
+
HashParser.parse_hash_of_pattern_lists(@hash, :nodes_by_config) : {}
|
47
|
+
end
|
48
|
+
|
49
|
+
def exclude_nodes_by_config
|
50
|
+
@exclude_nodes_by_config ||= exclude_nodes_by_config_valid? ?
|
51
|
+
HashParser.parse_hash_of_pattern_lists(@hash, :exclude_nodes_by_config) : {}
|
52
|
+
end
|
53
|
+
|
54
|
+
def roles
|
55
|
+
@roles ||= roles_valid? ?
|
56
|
+
HashParser.parse_pattern_list(@hash, :roles) : []
|
57
|
+
end
|
58
|
+
|
59
|
+
def exclude_roles
|
60
|
+
@exclude_roles ||= exclude_roles_valid? ?
|
61
|
+
HashParser.parse_pattern_list(@hash, :exclude_roles) : []
|
62
|
+
end
|
63
|
+
|
64
|
+
def commands
|
65
|
+
@commands ||= commands_valid? ? create_commands : nil
|
66
|
+
end
|
67
|
+
|
68
|
+
def set_plugin_defaults
|
69
|
+
@set_plugin_defaults ||= set_plugin_defaults_valid? ?
|
70
|
+
parse_plugin_pattern_array(:set_plugin_defaults) : []
|
71
|
+
end
|
72
|
+
|
73
|
+
def delete_plugin_defaults
|
74
|
+
@delete_plugin_defaults ||= delete_plugin_defaults_valid? ?
|
75
|
+
parse_plugin_pattern_array(:delete_plugin_defaults) : []
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
def nodes_valid?
|
81
|
+
pattern_list_valid?(@hash, :nodes)
|
82
|
+
end
|
83
|
+
|
84
|
+
def exclude_nodes_valid?
|
85
|
+
pattern_list_valid?(@hash, :exclude_nodes)
|
86
|
+
end
|
87
|
+
|
88
|
+
def nodes_by_config_valid?
|
89
|
+
hash_of_pattern_lists_valid?(@hash, :nodes_by_config)
|
90
|
+
end
|
91
|
+
|
92
|
+
def exclude_nodes_by_config_valid?
|
93
|
+
hash_of_pattern_lists_valid?(@hash, :exclude_nodes_by_config)
|
94
|
+
end
|
95
|
+
|
96
|
+
def roles_valid?
|
97
|
+
pattern_list_valid?(@hash, :roles)
|
98
|
+
end
|
99
|
+
|
100
|
+
def exclude_roles_valid?
|
101
|
+
pattern_list_valid?(@hash, :exclude_roles)
|
102
|
+
end
|
103
|
+
|
104
|
+
def pattern_list_valid?(hash, key, optional = true)
|
105
|
+
HashParser.pattern_list_valid?(hash, key, optional)
|
106
|
+
rescue PlanParsingError => e
|
107
|
+
raise PlanParsingError, "Step #{@name}: #{e.message}"
|
108
|
+
end
|
109
|
+
|
110
|
+
def hash_of_pattern_lists_valid?(hash, key, optional = true)
|
111
|
+
HashParser.hash_of_pattern_lists_valid?(hash, key, optional)
|
112
|
+
rescue PlanParsingError => e
|
113
|
+
raise PlanParsingError, "Step #{@name}: #{e.message}"
|
114
|
+
end
|
115
|
+
|
116
|
+
def commands_valid?
|
117
|
+
@hash[:commands] or
|
118
|
+
raise PlanParsingError, "Step #{@name}: A commands key has to be defined"
|
119
|
+
case @hash[:commands]
|
120
|
+
when String, Hash
|
121
|
+
true
|
122
|
+
when Array
|
123
|
+
@hash[:commands].all?{|c| c.kind_of?(String) or c.kind_of?(Hash)} or
|
124
|
+
raise PlanParsingError, "Step #{@name}: All commands must be Strings or Hashes"
|
125
|
+
else
|
126
|
+
raise PlanParsingError,
|
127
|
+
"Step #{@name}: The value for commands has to be a string, a hash or an array"
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def create_commands
|
132
|
+
[@hash[:commands]].flatten.map do |command|
|
133
|
+
::DopCommon::Command.new(command)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def plugin_pattern_array_valid?(key)
|
138
|
+
@hash[key].all? do |entry|
|
139
|
+
entry.kind_of?(Hash) or
|
140
|
+
raise PlanParsingError, "Step #{@name}: Each entry in the '#{key}' array has to be a hash"
|
141
|
+
HashParser.key_aliases(entry, :plugins, ['plugins', :plugin, 'plugin'])
|
142
|
+
pattern_list_valid?(entry, :plugins, false)
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def set_plugin_defaults_valid?
|
147
|
+
return false if @hash[:set_plugin_defaults].nil? # is optional
|
148
|
+
@hash[:set_plugin_defaults].kind_of?(Array) or
|
149
|
+
raise PlanParsingError, "Step #{@name}: The value of 'set_plugin_defaults' has to be an array"
|
150
|
+
plugin_pattern_array_valid?(:set_plugin_defaults)
|
151
|
+
end
|
152
|
+
|
153
|
+
def delete_plugin_defaults_valid?
|
154
|
+
return false if @hash[:delete_plugin_defaults].nil? # is optional
|
155
|
+
@hash[:delete_plugin_defaults].kind_of?(Array) or
|
156
|
+
@hash[:delete_plugin_defaults].kind_of?(String) or
|
157
|
+
@hash[:delete_plugin_defaults].kind_of?(Symbol) or
|
158
|
+
raise PlanParsingError, "Step #{@name}: The value of 'delete_plugin_defaults' has to be an array or :all"
|
159
|
+
unless @hash[:delete_plugin_defaults].kind_of?(Array)
|
160
|
+
['all', 'All', 'ALL', :all].include?(@hash[:delete_plugin_defaults]) or
|
161
|
+
raise PlanParsingError, "Step #{@name}: The value of 'delete_plugin_defaults' has to be an array or :all"
|
162
|
+
else
|
163
|
+
return false unless plugin_pattern_array_valid?(:delete_plugin_defaults)
|
164
|
+
@hash[:delete_plugin_defaults].all? do |entry|
|
165
|
+
HashParser.key_aliases(entry, :delete_keys, ['delete_keys', :delete_key, 'delete_key'])
|
166
|
+
entry[:delete_keys].nil? and
|
167
|
+
raise PlanParsingError, "Step #{@name}: Each entry in the 'delete_plugin_defaults' array needs a valid value for 'delete_keys'"
|
168
|
+
entry[:delete_keys].kind_of?(Array) or
|
169
|
+
entry[:delete_keys].kind_of?(String) or
|
170
|
+
entry[:delete_keys].kind_of?(Symbol) or
|
171
|
+
raise PlanParsingError, "Step #{@name}: The value for 'delete_keys' in 'delete_plugin_defaults' has to be a string or an array"
|
172
|
+
if entry[:delete_keys].kind_of?(Array)
|
173
|
+
entry[:delete_keys].all?{|e| e.kind_of?(String) or e.kind_of?(Symbol)} or
|
174
|
+
raise PlanParsingError, "Step #{@name}: The elements in 'delete_keys' in 'delete_plugin_defaults' have to me strings or symbols"
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
true
|
179
|
+
end
|
180
|
+
|
181
|
+
def parse_plugin_pattern_array(key)
|
182
|
+
unless @hash[key].kind_of?(Array)
|
183
|
+
:all
|
184
|
+
else
|
185
|
+
@hash[key].map do |entry|
|
186
|
+
result = entry.dup
|
187
|
+
result[:plugins] = HashParser.parse_pattern_list(entry, :plugins)
|
188
|
+
delete_keys = case entry[:delete_keys]
|
189
|
+
when 'all', 'All', 'ALL', :all then :all
|
190
|
+
when String, Array then [entry[:delete_keys]].flatten
|
191
|
+
else nil
|
192
|
+
end
|
193
|
+
result[:delete_keys] = delete_keys if delete_keys
|
194
|
+
result
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
end
|
200
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
#
|
2
|
+
# DOP common step set hash parser
|
3
|
+
#
|
4
|
+
|
5
|
+
module DopCommon
|
6
|
+
class StepSet
|
7
|
+
include Validator
|
8
|
+
|
9
|
+
attr_reader :name
|
10
|
+
|
11
|
+
def initialize(name, steps_array)
|
12
|
+
@name = name
|
13
|
+
@steps_array = steps_array
|
14
|
+
end
|
15
|
+
|
16
|
+
def validate
|
17
|
+
log_validation_method(:steps_valid?)
|
18
|
+
try_validate_obj("StepSet #{name}: Can't validate the steps part because of a previous error"){steps}
|
19
|
+
end
|
20
|
+
|
21
|
+
def steps
|
22
|
+
@steps ||= steps_valid? ? create_steps : nil
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def steps_valid?
|
28
|
+
@steps_array.any? or
|
29
|
+
raise PlanParsingError, "StepSet #{name}: no steps defined"
|
30
|
+
@steps_array.all?{|s| s.kind_of?(Hash)} or
|
31
|
+
raise PlanParsingError, "StepSet #{name}: steps array must only contain hashes"
|
32
|
+
end
|
33
|
+
|
34
|
+
def create_steps
|
35
|
+
@steps_array.map do |hash|
|
36
|
+
::DopCommon::Step.new(hash)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
#
|
2
|
+
# This is log formatter which will log to different files based on the context
|
3
|
+
# which was set for the current thread. if no context was set then it will log
|
4
|
+
# to the 'all' context.
|
5
|
+
#
|
6
|
+
# This is used to separate log for different nodes even for logs which are
|
7
|
+
# generated in some external library which is not aware of the context.
|
8
|
+
#
|
9
|
+
module DopCommon
|
10
|
+
class ThreadContextLogger
|
11
|
+
def initialize(log_path, contexts, all = true)
|
12
|
+
@log_path = log_path
|
13
|
+
@contexts = contexts
|
14
|
+
@all = all
|
15
|
+
@mutex = Mutex.new
|
16
|
+
@loggers = {}
|
17
|
+
@threads = {}
|
18
|
+
|
19
|
+
FileUtils.mkdir_p(@log_path)
|
20
|
+
create
|
21
|
+
end
|
22
|
+
|
23
|
+
def create
|
24
|
+
@mutex.synchronize do
|
25
|
+
add('all') if @all
|
26
|
+
@contexts.each{|context| add(context)}
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def cleanup
|
31
|
+
@mutex.synchronize do
|
32
|
+
@contexts.each{|context| remove(context)}
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def log_context=(context)
|
37
|
+
@mutex.synchronize do
|
38
|
+
@threads[Thread.current.object_id.to_s] = context
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def add(context)
|
45
|
+
log_file = File.join(@log_path, context)
|
46
|
+
logger = Logger.new(log_file)
|
47
|
+
if context == 'all'
|
48
|
+
logger.formatter = Logger::Formatter.new
|
49
|
+
else
|
50
|
+
logger.formatter = formatter(context)
|
51
|
+
end
|
52
|
+
logger.level = ::Logger.const_get(DopCommon.config.log_level.upcase)
|
53
|
+
|
54
|
+
@loggers[context] = logger
|
55
|
+
DopCommon.add_log_junction(logger)
|
56
|
+
end
|
57
|
+
|
58
|
+
def remove(context)
|
59
|
+
logger = @loggers[context]
|
60
|
+
DopCommon.remove_log_junction(logger)
|
61
|
+
end
|
62
|
+
|
63
|
+
def formatter(context)
|
64
|
+
orig_formatter = Logger::Formatter.new
|
65
|
+
Proc.new do |severity, datetime, progname, msg|
|
66
|
+
@mutex.synchronize do
|
67
|
+
if context == @threads[Thread.current.object_id.to_s]
|
68
|
+
orig_formatter.call(severity, datetime, progname, msg)
|
69
|
+
else
|
70
|
+
nil
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
module DopCommon
|
2
|
+
module Utils
|
3
|
+
def sanitize_env(additional_env_vars = {})
|
4
|
+
{
|
5
|
+
'HOME' => ENV['HOME'],
|
6
|
+
'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
|
7
|
+
}.merge(additional_env_vars)
|
8
|
+
end
|
9
|
+
module_function :sanitize_env
|
10
|
+
|
11
|
+
class DataSize
|
12
|
+
include Validator
|
13
|
+
|
14
|
+
KIBIBYTE = 1024.0
|
15
|
+
MEBIBYTE = 1048576.0
|
16
|
+
GIBIBYTE = 1073741824.0
|
17
|
+
KILOBYTE = 1000.0
|
18
|
+
MEGABYTE = 1000000.0
|
19
|
+
GIGABYTE = 1000000000.0
|
20
|
+
|
21
|
+
def initialize(input)
|
22
|
+
@input ||= input
|
23
|
+
end
|
24
|
+
|
25
|
+
def validate
|
26
|
+
log_validation_method(:input_valid?)
|
27
|
+
end
|
28
|
+
|
29
|
+
def size
|
30
|
+
@size ||= input_valid? ? create_size : nil
|
31
|
+
end
|
32
|
+
alias_method :bytes, :size
|
33
|
+
alias_method :b, :size
|
34
|
+
|
35
|
+
def kibibytes
|
36
|
+
size / KIBIBYTE
|
37
|
+
end
|
38
|
+
alias_method :k, :kibibytes
|
39
|
+
|
40
|
+
def mebibytes
|
41
|
+
size / MEBIBYTE
|
42
|
+
end
|
43
|
+
alias_method :m, :mebibytes
|
44
|
+
|
45
|
+
def gibibytes
|
46
|
+
size / GIBIBYTE
|
47
|
+
end
|
48
|
+
alias_method :g, :gibibytes
|
49
|
+
|
50
|
+
def kilobytes
|
51
|
+
size / KILOBYTE
|
52
|
+
end
|
53
|
+
alias_method :kb, :kilobytes
|
54
|
+
|
55
|
+
def megabytes
|
56
|
+
size / MEGABYTE
|
57
|
+
end
|
58
|
+
alias_method :mb, :megabytes
|
59
|
+
|
60
|
+
def gigabytes
|
61
|
+
size / GIGABYTE
|
62
|
+
end
|
63
|
+
alias_method :gb, :gigabytes
|
64
|
+
|
65
|
+
def to_s
|
66
|
+
size.to_s
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
|
71
|
+
def input_valid?
|
72
|
+
raise PlanParsingError, "DataSize: Invalid input '#{@input}'. It must be an integer or string" unless
|
73
|
+
[String, Fixnum].include?(@input.class)
|
74
|
+
raise PlanParsingError, "DataSize: Invalid input '#{@input}'. It must be greater than zero" if
|
75
|
+
@input.kind_of?(Fixnum) && @input < 1
|
76
|
+
raise PlanParsingError, "DataSize: Invalid input '#{@input}'. " \
|
77
|
+
"It must be a positive number followed by one of K,KB,M,MB,G,GB literals" if
|
78
|
+
@input.kind_of?(String) && @input !~ /^(([1-9]\d*)(\.\d+)?|0\.(0*[1-9]\d*))[KMG]B?$/
|
79
|
+
true
|
80
|
+
end
|
81
|
+
|
82
|
+
def create_size
|
83
|
+
if @input.kind_of?(String)
|
84
|
+
if @input.index(/K$/)
|
85
|
+
s = @input.sub(/K$/, '').to_f * KIBIBYTE
|
86
|
+
elsif @input.index(/M$/)
|
87
|
+
s = @input.sub(/M$/, '').to_f * MEBIBYTE
|
88
|
+
elsif @input.index(/G$/)
|
89
|
+
s = @input.sub(/G$/, '').to_f * GIBIBYTE
|
90
|
+
elsif @input.index(/KB$/)
|
91
|
+
s = @input.sub(/KB$/, '').to_f * KILOBYTE
|
92
|
+
elsif @input.index(/MB$/)
|
93
|
+
s = @input.sub(/MB$/, '').to_f * MEGABYTE
|
94
|
+
elsif @input.index(/GB$/)
|
95
|
+
s = @input.sub(/GB$/, '').to_f * GIGABYTE
|
96
|
+
else
|
97
|
+
s = @input
|
98
|
+
end
|
99
|
+
@size = s.to_i
|
100
|
+
else
|
101
|
+
@size = @input
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
#
|
2
|
+
# DOP Common Validator
|
3
|
+
#
|
4
|
+
# Some Validation Helper stuff
|
5
|
+
#
|
6
|
+
|
7
|
+
module DopCommon
|
8
|
+
module Validator
|
9
|
+
|
10
|
+
def valid?
|
11
|
+
@validity = true
|
12
|
+
validate
|
13
|
+
@validity
|
14
|
+
end
|
15
|
+
|
16
|
+
def set_not_valid
|
17
|
+
@validity = false
|
18
|
+
end
|
19
|
+
|
20
|
+
def log_validation_method(method, error_klass = PlanParsingError)
|
21
|
+
begin
|
22
|
+
send(method)
|
23
|
+
rescue error_klass => e
|
24
|
+
set_not_valid
|
25
|
+
DopCommon.log.error(e.message)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def try_validate_obj(message, error_klass = PlanParsingError)
|
30
|
+
begin
|
31
|
+
obj = yield
|
32
|
+
if obj.kind_of?(Array)
|
33
|
+
obj.each do |x|
|
34
|
+
x.validate
|
35
|
+
set_not_valid unless x.valid?
|
36
|
+
end
|
37
|
+
elsif obj.kind_of?(Hash)
|
38
|
+
obj.each_value do |x|
|
39
|
+
x.validate
|
40
|
+
set_not_valid unless x.valid?
|
41
|
+
end
|
42
|
+
else
|
43
|
+
obj.validate
|
44
|
+
set_not_valid unless obj.valid?
|
45
|
+
end
|
46
|
+
rescue error_klass => e
|
47
|
+
set_not_valid
|
48
|
+
DopCommon.log.warn(message)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
data/lib/dop_common.rb
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
require "dop_common/version"
|
2
|
+
require "dop_common/config"
|
3
|
+
require "dop_common/log"
|
4
|
+
require "dop_common/thread_context_logger"
|
5
|
+
require "dop_common/validator"
|
6
|
+
require "dop_common/hash_parser"
|
7
|
+
require "dop_common/utils"
|
8
|
+
require "dop_common/run_options"
|
9
|
+
require "dop_common/plan"
|
10
|
+
require 'dop_common/infrastructure'
|
11
|
+
require 'dop_common/network'
|
12
|
+
require 'dop_common/affinity_group'
|
13
|
+
require 'dop_common/hooks'
|
14
|
+
require 'dop_common/node'
|
15
|
+
require 'dop_common/infrastructure_properties'
|
16
|
+
require 'dop_common/interface'
|
17
|
+
require 'dop_common/step'
|
18
|
+
require 'dop_common/step_set'
|
19
|
+
require 'dop_common/configuration'
|
20
|
+
require 'dop_common/credential'
|
21
|
+
require 'dop_common/command'
|
22
|
+
require 'dop_common/pre_processor'
|
23
|
+
require 'dop_common/plan_store'
|
24
|
+
require 'dop_common/plan_cache'
|
25
|
+
require 'dop_common/state_store'
|
26
|
+
require 'dop_common/dns'
|
27
|
+
require 'dop_common/data_disk'
|
28
|
+
require 'dop_common/node_filter'
|
29
|
+
require 'dop_common/signal_handler'
|
30
|
+
|
31
|
+
module DopCommon
|
32
|
+
end
|