erchef-expander 11.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,150 @@
1
+ #
2
+ # Author:: Daniel DeLeo (<dan@opscode.com>)
3
+ # Copyright:: Copyright (c) 2011 Opscode, Inc.
4
+ # License:: Apache License, Version 2.0
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ require 'etc'
20
+ require 'chef/expander/loggable'
21
+
22
+ module Chef
23
+ module Expander
24
+
25
+ class AlreadyRunning < RuntimeError
26
+ end
27
+
28
+ class NoSuchUser < ArgumentError
29
+ end
30
+
31
+ class NoSuchGroup < ArgumentError
32
+ end
33
+
34
+ module Daemonizable
35
+ include Loggable
36
+
37
+ # Daemonizes the process if configured to do so, and ensures that only one
38
+ # copy of the process is running with a given config by obtaining an
39
+ # exclusive lock on the pidfile. Also sets process user and group if so
40
+ # configured.
41
+ # ===Raises
42
+ # * AlreadyRunning::: when another process has the exclusive lock on the pidfile
43
+ # * NoSuchUser::: when a user is configured that doesn't exist
44
+ # * NoSuchGroup::: when a group is configured that doesn't exist
45
+ # * SystemCallError::: if there is an error creating the pidfile
46
+ def configure_process
47
+ Expander.config.daemonize? ? daemonize : ensure_exclusive
48
+ set_user_and_group
49
+ end
50
+
51
+ def daemonize
52
+ acquire_locks
53
+ exit if fork
54
+ Process.setsid
55
+ exit if fork
56
+ write_pid
57
+ Dir.chdir('/')
58
+ STDIN.reopen("/dev/null")
59
+ STDOUT.reopen("/dev/null", "a")
60
+ STDERR.reopen("/dev/null", "a")
61
+ end
62
+
63
+ # When not forking into the background, this ensures only one chef-expander
64
+ # is running with a given config and writes the process id to the pidfile.
65
+ def ensure_exclusive
66
+ acquire_locks
67
+ write_pid
68
+ end
69
+
70
+ def set_user_and_group
71
+ return nil if Expander.config.user.nil?
72
+
73
+ if Expander.config.group.nil?
74
+ log.info {"Changing user to #{Expander.config.user}"}
75
+ else
76
+ log.info {"Changing user to #{Expander.config.user} and group to #{Expander.config.group}"}
77
+ end
78
+
79
+ unless (set_group && set_user)
80
+ log.error {"Unable to change user to #{Expander.config.user} - Are you root?"}
81
+ end
82
+ end
83
+
84
+ # Deletes the pidfile, releasing the exclusive lock on it in the process.
85
+ def release_locks
86
+ File.unlink(@pidfile.path) if File.exist?(@pidfile.path)
87
+ @pidfile.close unless @pidfile.closed?
88
+ end
89
+
90
+ private
91
+
92
+ def set_user
93
+ Process::Sys.setuid(target_uid)
94
+ true
95
+ rescue Errno::EPERM => e
96
+ log.debug {e}
97
+ false
98
+ end
99
+
100
+ def set_group
101
+ if gid = target_uid
102
+ Process::Sys.setgid(gid)
103
+ end
104
+ true
105
+ rescue Errno::EPERM
106
+ log.debug {e}
107
+ false
108
+ end
109
+
110
+ def target_uid
111
+ user = Expander.config.user
112
+ user.kind_of?(Fixnum) ? user : Etc.getpwnam(user).uid
113
+ rescue ArgumentError => e
114
+ log.debug {e}
115
+ raise NoSuchUser, "Cannot change user to #{user} - failed to find the uid"
116
+ end
117
+
118
+ def target_gid
119
+ if group = Expander.config.group
120
+ group.kind_of?(Fixnum) ? group : Etc.getgrnam(group).gid
121
+ else
122
+ nil
123
+ end
124
+ rescue ArgumentError => e
125
+ log.debug {e}
126
+ raise NoSuchGroup, "Cannot change group to #{group} - failed to find the gid"
127
+ end
128
+
129
+ def acquire_locks
130
+ @pidfile = File.open(Expander.config.pidfile, File::RDWR|File::CREAT, 0644)
131
+ unless @pidfile.flock(File::LOCK_EX | File::LOCK_NB)
132
+ pid = @pidfile.read.strip
133
+ msg = "Another instance of chef-expander (pid: #{pid}) has a lock on the pidfile (#{Expander.config.pidfile}). \n"\
134
+ "Configure a different pidfile to run multiple instances of chef-expander at once."
135
+ raise AlreadyRunning, msg
136
+ end
137
+ rescue Exception
138
+ @pidfile.close if @pidfile && !@pidfile.closed?
139
+ raise
140
+ end
141
+
142
+ def write_pid
143
+ @pidfile.truncate(0)
144
+ @pidfile.print("#{Process.pid}\n")
145
+ @pidfile.flush
146
+ end
147
+
148
+ end
149
+ end
150
+ end
@@ -0,0 +1,79 @@
1
+ #
2
+ # Author:: Daniel DeLeo (<dan@opscode.com>)
3
+ # Author:: Seth Falcon (<seth@opscode.com>)
4
+ # Author:: Chris Walters (<cw@opscode.com>)
5
+ # Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
6
+ # License:: Apache License, Version 2.0
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License");
9
+ # you may not use this file except in compliance with the License.
10
+ # You may obtain a copy of the License at
11
+ #
12
+ # http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS,
16
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ # See the License for the specific language governing permissions and
18
+ # limitations under the License.
19
+ #
20
+
21
+ require 'chef/expander/configuration'
22
+
23
+ module Chef
24
+ module Expander
25
+ # Flattens and expands nested Hashes representing Chef objects
26
+ # (e.g, Nodes, Roles, DataBagItems, etc.) into flat Hashes so the
27
+ # objects are suitable to be saved into Solr. This code is more or
28
+ # less copy-pasted from chef/solr/index which may or may not be a
29
+ # great idea, though that does minimize the dependencies and
30
+ # hopefully minimize the memory use of chef-expander.
31
+ class Flattener
32
+ UNDERSCORE = '_'
33
+ X = 'X'
34
+
35
+ X_CHEF_id_CHEF_X = 'X_CHEF_id_CHEF_X'
36
+ X_CHEF_database_CHEF_X = 'X_CHEF_database_CHEF_X'
37
+ X_CHEF_type_CHEF_X = 'X_CHEF_type_CHEF_X'
38
+
39
+ def initialize(item)
40
+ @item = item
41
+ end
42
+
43
+ def flattened_item
44
+ @flattened_item || flatten_and_expand
45
+ end
46
+
47
+ def flatten_and_expand
48
+ @flattened_item = Hash.new {|hash, key| hash[key] = []}
49
+
50
+ @item.each do |key, value|
51
+ flatten_each([key.to_s], value)
52
+ end
53
+
54
+ @flattened_item.each_value { |values| values.uniq! }
55
+ @flattened_item
56
+ end
57
+
58
+ def flatten_each(keys, values)
59
+ case values
60
+ when Hash
61
+ values.each do |child_key, child_value|
62
+ add_field_value(keys, child_key)
63
+ flatten_each(keys + [child_key.to_s], child_value)
64
+ end
65
+ when Array
66
+ values.each { |child_value| flatten_each(keys, child_value) }
67
+ else
68
+ add_field_value(keys, values)
69
+ end
70
+ end
71
+
72
+ def add_field_value(keys, value)
73
+ value = value.to_s
74
+ @flattened_item[keys.join(UNDERSCORE)] << value
75
+ @flattened_item[keys.last] << value
76
+ end
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,40 @@
1
+ #
2
+ # Author:: Daniel DeLeo (<dan@opscode.com>)
3
+ # Author:: Seth Falcon (<seth@opscode.com>)
4
+ # Author:: Chris Walters (<cw@opscode.com>)
5
+ # Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
6
+ # License:: Apache License, Version 2.0
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License");
9
+ # you may not use this file except in compliance with the License.
10
+ # You may obtain a copy of the License at
11
+ #
12
+ # http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS,
16
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ # See the License for the specific language governing permissions and
18
+ # limitations under the License.
19
+ #
20
+
21
+ require 'chef/expander/logger'
22
+ require 'mixlib/log'
23
+
24
+ module Chef
25
+ module Expander
26
+ module Loggable
27
+
28
+ # TODO: it's admittedly janky to set up the default logging this way.
29
+ STDOUT.sync = true
30
+ LOGGER = Logger.new(STDOUT)
31
+ LOGGER.level = :debug
32
+
33
+ def log
34
+ LOGGER
35
+ end
36
+
37
+ end
38
+ end
39
+ end
40
+
@@ -0,0 +1,135 @@
1
+ #
2
+ # Author:: Daniel DeLeo (<dan@opscode.com>)
3
+ # Copyright:: Copyright (c) 2011 Opscode, Inc.
4
+ # License:: Apache License, Version 2.0
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ require 'logger'
20
+
21
+ module Chef
22
+ module Expander
23
+
24
+ class InvalidLogDevice < ArgumentError
25
+ end
26
+
27
+ class InvalidLogLevel < ArgumentError
28
+ end
29
+
30
+ # Customized Logger class that dispenses with the unnecessary mutexing.
31
+ # As long as you write one line at a time, the OS will take care of keeping
32
+ # your output in order. Expander commonly runs as a cluster of worker
33
+ # processes so the mutexing wasn't actually helping us anyway.
34
+ #
35
+ # We don't use the program name field in the logger, so support for that
36
+ # has been removed. The log format is also hardcoded since we don't ever
37
+ # change the format.
38
+ class Logger < ::Logger
39
+
40
+ LEVELS = { :debug=>DEBUG, :info=>INFO, :warn=>WARN, :error=>ERROR, :fatal=>FATAL}
41
+ LEVEL_INTEGERS = LEVELS.invert
42
+ LEVEL_TO_STR = Hash[LEVEL_INTEGERS.map {|i,sym| [i,sym.to_s.upcase]}]
43
+
44
+ LOG_DEVICES = []
45
+
46
+ at_exit do
47
+ LOG_DEVICES.each {|io| io.close if io.respond_to?(:closed?) && !io.closed?}
48
+ end
49
+
50
+ attr_reader :log_device
51
+
52
+ # (re-)initialize the Logger with a new IO object or file to log to.
53
+ def init(log_device)
54
+ @log_device = initialize_log_device(log_device)
55
+ end
56
+
57
+ def initialize(log_device)
58
+ @level = DEBUG
59
+ init(log_device)
60
+ end
61
+
62
+ def level=(new_level)
63
+ @level = if new_level.kind_of?(Fixnum) && LEVEL_INTEGERS.key?(new_level)
64
+ new
65
+ elsif LEVELS.key?(new_level)
66
+ LEVELS[new_level]
67
+ else
68
+ raise InvalidLogLevel, "#{new_level} is not a valid log level. Valid log levels are [#{LEVEL_INTEGERS.keys.join(',')}] and [#{LEVELS.join(',')}]"
69
+ end
70
+ end
71
+
72
+ def <<(msg)
73
+ @log_device.print(msg)
74
+ end
75
+
76
+ def add(severity=UNKNOWN, message = nil, progname = nil, &block)
77
+ return true unless severity >= @level
78
+
79
+ message ||= progname # level methods (e.g, #debug) pass explicit message as progname
80
+
81
+ if message.nil? && block_given?
82
+ message = yield
83
+ end
84
+
85
+ self << sprintf("[%s] %s: %s\n", Time.new.rfc2822(), LEVEL_TO_STR[severity], msg2str(message))
86
+ true
87
+ end
88
+
89
+ alias :log :add
90
+
91
+ private
92
+
93
+ def msg2str(msg)
94
+ case msg
95
+ when ::String
96
+ msg
97
+ when ::Exception
98
+ "#{ msg.message } (#{ msg.class })\n" <<
99
+ (msg.backtrace || []).join("\n")
100
+ else
101
+ msg.inspect
102
+ end
103
+ end
104
+
105
+ def logging_at_severity?(severity=nil)
106
+ end
107
+
108
+ def initialize_log_device(dev)
109
+ unless dev.respond_to? :sync=
110
+ assert_valid_path!(dev)
111
+ dev = File.open(dev.to_str, "a")
112
+ LOG_DEVICES << dev
113
+ end
114
+
115
+ dev.sync = true
116
+ dev
117
+ end
118
+
119
+ def assert_valid_path!(path)
120
+ enclosing_directory = File.dirname(path)
121
+ unless File.directory?(enclosing_directory)
122
+ raise InvalidLogDevice, "You must create the enclosing directory #{enclosing_directory} before the log file #{path} can be created."
123
+ end
124
+ if File.exist?(path)
125
+ unless File.writable?(path)
126
+ raise InvalidLogDevice, "The log file you specified (#{path}) is not writable by user #{Process.euid}"
127
+ end
128
+ elsif !File.writable?(enclosing_directory)
129
+ raise InvalidLogDevice, "You specified a log file #{path} but user #{Process.euid} is not permitted to create files there."
130
+ end
131
+ end
132
+
133
+ end
134
+ end
135
+ end
@@ -0,0 +1,177 @@
1
+ #
2
+ # Author:: Daniel DeLeo (<dan@opscode.com>)
3
+ # Author:: Seth Falcon (<seth@opscode.com>)
4
+ # Author:: Chris Walters (<cw@opscode.com>)
5
+ # Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
6
+ # License:: Apache License, Version 2.0
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License");
9
+ # you may not use this file except in compliance with the License.
10
+ # You may obtain a copy of the License at
11
+ #
12
+ # http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS,
16
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ # See the License for the specific language governing permissions and
18
+ # limitations under the License.
19
+ #
20
+
21
+ require 'uuidtools'
22
+ require 'amqp'
23
+ require 'mq'
24
+ require 'open3'
25
+
26
+ require 'chef/expander/loggable'
27
+
28
+ module Chef
29
+ module Expander
30
+ class Node
31
+
32
+ include Loggable
33
+
34
+ def self.from_hash(node_info)
35
+ new(node_info[:guid], node_info[:hostname_f], node_info[:pid])
36
+ end
37
+
38
+ def self.local_node
39
+ new(guid, hostname_f, Process.pid)
40
+ end
41
+
42
+ def self.guid
43
+ return @guid if @guid
44
+ @guid = UUIDTools::UUID.random_create.to_s
45
+ end
46
+
47
+ def self.hostname_f
48
+ @hostname ||= Open3.popen3("hostname -f") {|stdin, stdout, stderr| stdout.read }.strip
49
+ end
50
+
51
+ attr_reader :guid
52
+
53
+ attr_reader :hostname_f
54
+
55
+ attr_reader :pid
56
+
57
+ def initialize(guid, hostname_f, pid)
58
+ @guid, @hostname_f, @pid = guid, hostname_f, pid
59
+ end
60
+
61
+ def start(&message_handler)
62
+ attach_to_queue(exclusive_control_queue, "exclusive control", &message_handler)
63
+ attach_to_queue(shared_control_queue, "shared_control", &message_handler)
64
+ attach_to_queue(broadcast_control_queue, "broadcast control", &message_handler)
65
+ end
66
+
67
+ def attach_to_queue(queue, colloquial_name, &message_handler)
68
+ queue.subscribe(:ack => true) do |headers, payload|
69
+ log.debug { "received message on #{colloquial_name} queue: #{payload}" }
70
+ message_handler.call(payload)
71
+ headers.ack
72
+ end
73
+ end
74
+
75
+ def stop
76
+ log.debug { "unsubscribing from broadcast control queue"}
77
+ broadcast_control_queue.unsubscribe(:nowait => false)
78
+
79
+ log.debug { "unsubscribing from shared control queue" }
80
+ shared_control_queue.unsubscribe(:nowait => false)
81
+
82
+ log.debug { "unsubscribing from exclusive control queue" }
83
+ exclusive_control_queue.unsubscribe(:nowait => false)
84
+ end
85
+
86
+ def direct_message(message)
87
+ log.debug { "publishing direct message to node #{identifier}: #{message}" }
88
+ exclusive_control_queue.publish(message)
89
+ end
90
+
91
+ def shared_message(message)
92
+ log.debug { "publishing shared message #{message}"}
93
+ shared_control_queue.publish(message)
94
+ end
95
+
96
+ def broadcast_message(message)
97
+ log.debug { "publishing broadcast message #{message}" }
98
+ broadcast_control_exchange.publish(message)
99
+ end
100
+
101
+ # The exclusive control queue is for point-to-point messaging, i.e.,
102
+ # messages directly addressed to this node
103
+ def exclusive_control_queue
104
+ @exclusive_control_queue ||= begin
105
+ log.debug { "declaring exclusive control queue #{exclusive_control_queue_name}" }
106
+ MQ.queue(exclusive_control_queue_name)
107
+ end
108
+ end
109
+
110
+ # The shared control queue is for 1 to (1 of N) messaging, i.e.,
111
+ # messages that can go to any one node.
112
+ def shared_control_queue
113
+ @shared_control_queue ||= begin
114
+ log.debug { "declaring shared control queue #{shared_control_queue_name}" }
115
+ MQ.queue(shared_control_queue_name)
116
+ end
117
+ end
118
+
119
+ # The broadcast control queue is for 1 to N messaging, i.e.,
120
+ # messages that go to every node
121
+ def broadcast_control_queue
122
+ @broadcast_control_queue ||= begin
123
+ log.debug { "declaring broadcast control queue #{broadcast_control_queue_name}"}
124
+ q = MQ.queue(broadcast_control_queue_name)
125
+ log.debug { "binding broadcast control queue to broadcast control exchange"}
126
+ q.bind(broadcast_control_exchange)
127
+ q
128
+ end
129
+ end
130
+
131
+ def broadcast_control_exchange
132
+ @broadcast_control_exchange ||= begin
133
+ log.debug { "declaring broadcast control exchange opscode-platfrom-control--broadcast" }
134
+ MQ.fanout(broadcast_control_exchange_name, :nowait => false)
135
+ end
136
+ end
137
+
138
+ def shared_control_queue_name
139
+ SHARED_CONTROL_QUEUE_NAME
140
+ end
141
+
142
+ def broadcast_control_queue_name
143
+ @broadcast_control_queue_name ||= "#{identifier}--broadcast"
144
+ end
145
+
146
+ def broadcast_control_exchange_name
147
+ BROADCAST_CONTROL_EXCHANGE_NAME
148
+ end
149
+
150
+ def exclusive_control_queue_name
151
+ @exclusive_control_queue_name ||= "#{identifier}--exclusive-control"
152
+ end
153
+
154
+ def identifier
155
+ "#{hostname_f}--#{pid}--#{guid}"
156
+ end
157
+
158
+ def ==(other)
159
+ other.respond_to?(:guid) && other.respond_to?(:hostname_f) && other.respond_to?(:pid) &&
160
+ (other.guid == guid) && (other.hostname_f == hostname_f) && (other.pid == pid)
161
+ end
162
+
163
+ def eql?(other)
164
+ (other.class == self.class) && (other.hash == hash)
165
+ end
166
+
167
+ def hash
168
+ identifier.hash
169
+ end
170
+
171
+ def to_hash
172
+ {:guid => @guid, :hostname_f => @hostname_f, :pid => @pid}
173
+ end
174
+
175
+ end
176
+ end
177
+ end