erchef-expander 11.4.0

data/lib/chef/expander/daemonizable.rb

@@ -0,0 +1,150 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Copyright:: Copyright (c) 2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'etc'
+require 'chef/expander/loggable'
+
+module Chef
+  module Expander
+
+    class AlreadyRunning < RuntimeError
+    end
+
+    class NoSuchUser < ArgumentError
+    end
+
+    class NoSuchGroup < ArgumentError
+    end
+
+    module Daemonizable
+      include Loggable
+
+      # Daemonizes the process if configured to do so, and ensures that only one
+      # copy of the process is running with a given config by obtaining an
+      # exclusive lock on the pidfile. Also sets process user and group if so
+      # configured.
+      # ===Raises
+      # * AlreadyRunning::: when another process has the exclusive lock on the pidfile
+      # * NoSuchUser::: when a user is configured that doesn't exist
+      # * NoSuchGroup::: when a group is configured that doesn't exist
+      # * SystemCallError::: if there is an error creating the pidfile
+      def configure_process
+        Expander.config.daemonize? ? daemonize : ensure_exclusive
+        set_user_and_group
+      end
+
+      def daemonize
+        acquire_locks
+        exit if fork
+        Process.setsid
+        exit if fork
+        write_pid
+        Dir.chdir('/')
+        STDIN.reopen("/dev/null")
+        STDOUT.reopen("/dev/null", "a")
+        STDERR.reopen("/dev/null", "a")
+      end
+
+      # When not forking into the background, this ensures only one chef-expander
+      # is running with a given config and writes the process id to the pidfile.
+      def ensure_exclusive
+        acquire_locks
+        write_pid
+      end
+
+      def set_user_and_group
+        return nil if Expander.config.user.nil?
+
+        if Expander.config.group.nil?
+          log.info {"Changing user to #{Expander.config.user}"}
+        else
+          log.info {"Changing user to #{Expander.config.user} and group to #{Expander.config.group}"}
+        end
+
+        unless (set_group && set_user)
+          log.error {"Unable to change user to #{Expander.config.user} - Are you root?"}
+        end
+      end
+
+      # Deletes the pidfile, releasing the exclusive lock on it in the process.
+      def release_locks
+        File.unlink(@pidfile.path) if File.exist?(@pidfile.path)
+        @pidfile.close unless @pidfile.closed?
+      end
+
+      private
+
+      def set_user
+        Process::Sys.setuid(target_uid)
+        true
+      rescue Errno::EPERM => e
+        log.debug {e}
+        false
+      end
+
+      def set_group
+        if gid = target_uid
+          Process::Sys.setgid(gid)
+        end
+        true
+      rescue Errno::EPERM
+        log.debug {e}
+        false
+      end
+
+      def target_uid
+        user = Expander.config.user
+        user.kind_of?(Fixnum) ? user : Etc.getpwnam(user).uid
+      rescue ArgumentError => e
+        log.debug {e}
+        raise NoSuchUser, "Cannot change user to #{user} - failed to find the uid"
+      end
+
+      def target_gid
+        if group = Expander.config.group
+          group.kind_of?(Fixnum) ? group : Etc.getgrnam(group).gid
+        else
+          nil
+        end
+      rescue ArgumentError => e
+        log.debug {e}
+        raise NoSuchGroup, "Cannot change group to #{group} - failed to find the gid"
+      end
+
+      def acquire_locks
+        @pidfile = File.open(Expander.config.pidfile, File::RDWR|File::CREAT, 0644)
+        unless @pidfile.flock(File::LOCK_EX | File::LOCK_NB)
+          pid = @pidfile.read.strip
+          msg = "Another instance of chef-expander (pid: #{pid}) has a lock on the pidfile (#{Expander.config.pidfile}). \n"\
+                "Configure a different pidfile to run multiple instances of chef-expander at once."
+          raise AlreadyRunning, msg
+        end
+      rescue Exception
+        @pidfile.close if @pidfile && !@pidfile.closed?
+        raise
+      end
+
+      def write_pid
+        @pidfile.truncate(0)
+        @pidfile.print("#{Process.pid}\n")
+        @pidfile.flush
+      end
+
+    end
+  end
+end

data/lib/chef/expander/flattener.rb

@@ -0,0 +1,79 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/expander/configuration'
+
+module Chef
+  module Expander
+    # Flattens and expands nested Hashes representing Chef objects
+    # (e.g, Nodes, Roles, DataBagItems, etc.) into flat Hashes so the
+    # objects are suitable to be saved into Solr. This code is more or
+    # less copy-pasted from chef/solr/index which may or may not be a
+    # great idea, though that does minimize the dependencies and
+    # hopefully minimize the memory use of chef-expander.
+    class Flattener
+      UNDERSCORE              = '_'
+      X                       = 'X'
+
+      X_CHEF_id_CHEF_X        = 'X_CHEF_id_CHEF_X'
+      X_CHEF_database_CHEF_X  = 'X_CHEF_database_CHEF_X'
+      X_CHEF_type_CHEF_X      = 'X_CHEF_type_CHEF_X'
+
+      def initialize(item)
+        @item = item
+      end
+
+      def flattened_item
+        @flattened_item || flatten_and_expand
+      end
+
+      def flatten_and_expand
+        @flattened_item = Hash.new {|hash, key| hash[key] = []}
+
+        @item.each do |key, value|
+          flatten_each([key.to_s], value)
+        end
+
+        @flattened_item.each_value { |values| values.uniq! }
+        @flattened_item
+      end
+
+      def flatten_each(keys, values)
+        case values
+        when Hash
+          values.each do |child_key, child_value|
+            add_field_value(keys, child_key)
+            flatten_each(keys + [child_key.to_s], child_value)
+          end
+        when Array
+          values.each { |child_value| flatten_each(keys, child_value) }
+        else
+          add_field_value(keys, values)
+        end
+      end
+
+      def add_field_value(keys, value)
+        value = value.to_s
+        @flattened_item[keys.join(UNDERSCORE)] << value
+        @flattened_item[keys.last] << value
+      end
+    end
+  end
+end

data/lib/chef/expander/loggable.rb

@@ -0,0 +1,40 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/expander/logger'
+require 'mixlib/log'
+
+module Chef
+  module Expander
+    module Loggable
+
+      # TODO: it's admittedly janky to set up the default logging this way.
+      STDOUT.sync = true
+      LOGGER = Logger.new(STDOUT)
+      LOGGER.level = :debug
+
+      def log
+        LOGGER
+      end
+
+    end
+  end
+end
+

data/lib/chef/expander/logger.rb

@@ -0,0 +1,135 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Copyright:: Copyright (c) 2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'logger'
+
+module Chef
+  module Expander
+
+    class InvalidLogDevice < ArgumentError
+    end
+
+    class InvalidLogLevel < ArgumentError
+    end
+
+    # Customized Logger class that dispenses with the unnecessary mutexing.
+    # As long as you write one line at a time, the OS will take care of keeping
+    # your output in order. Expander commonly runs as a cluster of worker 
+    # processes so the mutexing wasn't actually helping us anyway.
+    #
+    # We don't use the program name field in the logger, so support for that
+    # has been removed. The log format is also hardcoded since we don't ever
+    # change the format.
+    class Logger < ::Logger
+
+      LEVELS = { :debug=>DEBUG, :info=>INFO, :warn=>WARN, :error=>ERROR, :fatal=>FATAL}
+      LEVEL_INTEGERS = LEVELS.invert
+      LEVEL_TO_STR = Hash[LEVEL_INTEGERS.map {|i,sym| [i,sym.to_s.upcase]}]
+
+      LOG_DEVICES = []
+
+      at_exit do
+        LOG_DEVICES.each {|io| io.close if io.respond_to?(:closed?) && !io.closed?}
+      end
+
+      attr_reader :log_device
+
+      # (re-)initialize the Logger with a new IO object or file to log to.
+      def init(log_device)
+        @log_device = initialize_log_device(log_device)
+      end
+
+      def initialize(log_device)
+        @level = DEBUG
+        init(log_device)
+      end
+
+      def level=(new_level)
+        @level =  if new_level.kind_of?(Fixnum) && LEVEL_INTEGERS.key?(new_level)
+          new
+        elsif LEVELS.key?(new_level)
+          LEVELS[new_level]
+        else
+          raise InvalidLogLevel, "#{new_level} is not a valid log level. Valid log levels are [#{LEVEL_INTEGERS.keys.join(',')}] and [#{LEVELS.join(',')}]"
+        end
+      end
+
+      def <<(msg)
+        @log_device.print(msg)
+      end
+
+      def add(severity=UNKNOWN, message = nil, progname = nil, &block)
+        return true unless severity >= @level
+
+        message ||= progname # level methods (e.g, #debug) pass explicit message as progname
+
+        if message.nil? && block_given?
+          message = yield
+        end
+
+        self << sprintf("[%s] %s: %s\n", Time.new.rfc2822(), LEVEL_TO_STR[severity], msg2str(message))
+        true
+      end
+
+      alias :log :add
+
+      private
+
+      def msg2str(msg)
+        case msg
+        when ::String
+          msg
+        when ::Exception
+          "#{ msg.message } (#{ msg.class })\n" <<
+            (msg.backtrace || []).join("\n")
+        else
+          msg.inspect
+        end
+      end
+
+      def logging_at_severity?(severity=nil)
+      end
+
+      def initialize_log_device(dev)
+        unless dev.respond_to? :sync=
+          assert_valid_path!(dev)
+          dev = File.open(dev.to_str, "a")
+          LOG_DEVICES << dev
+        end
+
+        dev.sync = true
+        dev
+      end
+
+      def assert_valid_path!(path)
+        enclosing_directory = File.dirname(path)
+        unless File.directory?(enclosing_directory)
+          raise InvalidLogDevice, "You must create the enclosing directory #{enclosing_directory} before the log file #{path} can be created."
+        end
+        if File.exist?(path)
+          unless File.writable?(path)
+            raise InvalidLogDevice, "The log file you specified (#{path}) is not writable by user #{Process.euid}"
+          end
+        elsif !File.writable?(enclosing_directory)
+          raise InvalidLogDevice, "You specified a log file #{path} but user #{Process.euid} is not permitted to create files there."
+        end
+      end
+
+    end
+  end
+end

data/lib/chef/expander/node.rb

@@ -0,0 +1,177 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'uuidtools'
+require 'amqp'
+require 'mq'
+require 'open3'
+
+require 'chef/expander/loggable'
+
+module Chef
+  module Expander
+    class Node
+
+      include Loggable
+
+      def self.from_hash(node_info)
+        new(node_info[:guid], node_info[:hostname_f], node_info[:pid])
+      end
+
+      def self.local_node
+        new(guid, hostname_f, Process.pid)
+      end
+
+      def self.guid
+        return @guid if @guid
+        @guid = UUIDTools::UUID.random_create.to_s
+      end
+
+      def self.hostname_f
+        @hostname ||= Open3.popen3("hostname -f") {|stdin, stdout, stderr| stdout.read }.strip
+      end
+
+      attr_reader :guid
+
+      attr_reader :hostname_f
+
+      attr_reader :pid
+
+      def initialize(guid, hostname_f, pid)
+        @guid, @hostname_f, @pid = guid, hostname_f, pid
+      end
+
+      def start(&message_handler)
+        attach_to_queue(exclusive_control_queue, "exclusive control", &message_handler)
+        attach_to_queue(shared_control_queue, "shared_control", &message_handler)
+        attach_to_queue(broadcast_control_queue, "broadcast control", &message_handler)
+      end
+
+      def attach_to_queue(queue, colloquial_name, &message_handler)
+        queue.subscribe(:ack => true) do |headers, payload|
+          log.debug { "received message on #{colloquial_name} queue: #{payload}" }
+          message_handler.call(payload)
+          headers.ack
+        end
+      end
+
+      def stop
+        log.debug { "unsubscribing from broadcast control queue"}
+        broadcast_control_queue.unsubscribe(:nowait => false)
+
+        log.debug { "unsubscribing from shared control queue" }
+        shared_control_queue.unsubscribe(:nowait => false)
+
+        log.debug { "unsubscribing from exclusive control queue" }
+        exclusive_control_queue.unsubscribe(:nowait => false)
+      end
+
+      def direct_message(message)
+        log.debug { "publishing direct message to node #{identifier}: #{message}" }
+        exclusive_control_queue.publish(message)
+      end
+
+      def shared_message(message)
+        log.debug { "publishing shared message #{message}"}
+        shared_control_queue.publish(message)
+      end
+
+      def broadcast_message(message)
+        log.debug { "publishing broadcast message #{message}" }
+        broadcast_control_exchange.publish(message)
+      end
+
+      # The exclusive control queue is for point-to-point messaging, i.e.,
+      # messages directly addressed to this node
+      def exclusive_control_queue
+        @exclusive_control_queue ||= begin
+          log.debug { "declaring exclusive control queue #{exclusive_control_queue_name}" }
+          MQ.queue(exclusive_control_queue_name)
+        end
+      end
+
+      # The shared control queue is for 1 to (1 of N) messaging, i.e.,
+      # messages that can go to any one node.
+      def shared_control_queue
+        @shared_control_queue ||= begin
+          log.debug { "declaring shared control queue #{shared_control_queue_name}" }
+          MQ.queue(shared_control_queue_name)
+        end
+      end
+
+      # The broadcast control queue is for 1 to N messaging, i.e.,
+      # messages that go to every node
+      def broadcast_control_queue
+        @broadcast_control_queue ||= begin
+          log.debug { "declaring broadcast control queue #{broadcast_control_queue_name}"}
+          q = MQ.queue(broadcast_control_queue_name)
+          log.debug { "binding broadcast control queue to broadcast control exchange"}
+          q.bind(broadcast_control_exchange)
+          q
+        end
+      end
+
+      def broadcast_control_exchange
+        @broadcast_control_exchange ||= begin
+          log.debug { "declaring broadcast control exchange opscode-platfrom-control--broadcast" }
+          MQ.fanout(broadcast_control_exchange_name, :nowait => false)
+        end
+      end
+
+      def shared_control_queue_name
+        SHARED_CONTROL_QUEUE_NAME
+      end
+
+      def broadcast_control_queue_name
+        @broadcast_control_queue_name ||= "#{identifier}--broadcast"
+      end
+
+      def broadcast_control_exchange_name
+        BROADCAST_CONTROL_EXCHANGE_NAME
+      end
+
+      def exclusive_control_queue_name
+        @exclusive_control_queue_name ||= "#{identifier}--exclusive-control"
+      end
+
+      def identifier
+        "#{hostname_f}--#{pid}--#{guid}"
+      end
+
+      def ==(other)
+        other.respond_to?(:guid) && other.respond_to?(:hostname_f) && other.respond_to?(:pid) &&
+        (other.guid == guid) && (other.hostname_f == hostname_f) && (other.pid == pid)
+      end
+
+      def eql?(other)
+        (other.class == self.class) && (other.hash == hash)
+      end
+
+      def hash
+        identifier.hash
+      end
+
+      def to_hash
+        {:guid => @guid, :hostname_f => @hostname_f, :pid => @pid}
+      end
+
+    end
+  end
+end