chef-expander 0.10.0.beta.0

data/lib/chef/expander/flattener.rb

@@ -0,0 +1,79 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/expander/configuration'
+
+module Chef
+  module Expander
+    # Flattens and expands nested Hashes representing Chef objects
+    # (e.g, Nodes, Roles, DataBagItems, etc.) into flat Hashes so the
+    # objects are suitable to be saved into Solr. This code is more or
+    # less copy-pasted from chef/solr/index which may or may not be a
+    # great idea, though that does minimize the dependencies and
+    # hopefully minimize the memory use of chef-expander.
+    class Flattener
+      UNDERSCORE              = '_'
+      X                       = 'X'
+
+      X_CHEF_id_CHEF_X        = 'X_CHEF_id_CHEF_X'
+      X_CHEF_database_CHEF_X  = 'X_CHEF_database_CHEF_X'
+      X_CHEF_type_CHEF_X      = 'X_CHEF_type_CHEF_X'
+
+      def initialize(item)
+        @item = item
+      end
+
+      def flattened_item
+        @flattened_item || flatten_and_expand
+      end
+
+      def flatten_and_expand
+        @flattened_item = Hash.new {|hash, key| hash[key] = []}
+
+        @item.each do |key, value|
+          flatten_each([key.to_s], value)
+        end
+
+        @flattened_item.each_value { |values| values.uniq! }
+        @flattened_item
+      end
+
+      def flatten_each(keys, values)
+        case values
+        when Hash
+          values.each do |child_key, child_value|
+            add_field_value(keys, child_key)
+            flatten_each(keys + [child_key.to_s], child_value)
+          end
+        when Array
+          values.each { |child_value| flatten_each(keys, child_value) }
+        else
+          add_field_value(keys, values)
+        end
+      end
+
+      def add_field_value(keys, value)
+        value = value.to_s
+        @flattened_item[keys.join(UNDERSCORE)] << value
+        @flattened_item[keys.last] << value
+      end
+    end
+  end
+end

data/lib/chef/expander/loggable.rb

@@ -0,0 +1,40 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/expander/logger'
+require 'mixlib/log'
+
+module Chef
+  module Expander
+    module Loggable
+
+      # TODO: it's admittedly janky to set up the default logging this way.
+      STDOUT.sync = true
+      LOGGER = Logger.new(STDOUT)
+      LOGGER.level = :debug
+
+      def log
+        LOGGER
+      end
+
+    end
+  end
+end
+

data/lib/chef/expander/logger.rb

@@ -0,0 +1,135 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Copyright:: Copyright (c) 2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'logger'
+
+module Chef
+  module Expander
+
+    class InvalidLogDevice < ArgumentError
+    end
+
+    class InvalidLogLevel < ArgumentError
+    end
+
+    # Customized Logger class that dispenses with the unnecessary mutexing.
+    # As long as you write one line at a time, the OS will take care of keeping
+    # your output in order. Expander commonly runs as a cluster of worker 
+    # processes so the mutexing wasn't actually helping us anyway.
+    #
+    # We don't use the program name field in the logger, so support for that
+    # has been removed. The log format is also hardcoded since we don't ever
+    # change the format.
+    class Logger < ::Logger
+
+      LEVELS = { :debug=>DEBUG, :info=>INFO, :warn=>WARN, :error=>ERROR, :fatal=>FATAL}
+      LEVEL_INTEGERS = LEVELS.invert
+      LEVEL_TO_STR = Hash[LEVEL_INTEGERS.map {|i,sym| [i,sym.to_s.upcase]}]
+
+      LOG_DEVICES = []
+
+      at_exit do
+        LOG_DEVICES.each {|io| io.close if io.respond_to?(:closed?) && !io.closed?}
+      end
+
+      attr_reader :log_device
+
+      # (re-)initialize the Logger with a new IO object or file to log to.
+      def init(log_device)
+        @log_device = initialize_log_device(log_device)
+      end
+
+      def initialize(log_device)
+        @level = DEBUG
+        init(log_device)
+      end
+
+      def level=(new_level)
+        @level =  if new_level.kind_of?(Fixnum) && LEVEL_INTEGERS.key?(new_level)
+          new
+        elsif LEVELS.key?(new_level)
+          LEVELS[new_level]
+        else
+          raise InvalidLogLevel, "#{new_level} is not a valid log level. Valid log levels are [#{LEVEL_INTEGERS.keys.join(',')}] and [#{LEVELS.join(',')}]"
+        end
+      end
+
+      def <<(msg)
+        @log_device.print(msg)
+      end
+
+      def add(severity=UNKNOWN, message = nil, progname = nil, &block)
+        return true unless severity >= @level
+
+        message ||= progname # level methods (e.g, #debug) pass explicit message as progname
+
+        if message.nil? && block_given?
+          message = yield
+        end
+
+        self << sprintf("[%s] %s: %s\n", Time.new.rfc2822(), LEVEL_TO_STR[severity], msg2str(message))
+        true
+      end
+
+      alias :log :add
+
+      private
+
+      def msg2str(msg)
+        case msg
+        when ::String
+          msg
+        when ::Exception
+          "#{ msg.message } (#{ msg.class })\n" <<
+            (msg.backtrace || []).join("\n")
+        else
+          msg.inspect
+        end
+      end
+
+      def logging_at_severity?(severity=nil)
+      end
+
+      def initialize_log_device(dev)
+        unless dev.respond_to? :sync=
+          assert_valid_path!(dev)
+          dev = File.open(dev.to_str, "a")
+          LOG_DEVICES << dev
+        end
+
+        dev.sync = true
+        dev
+      end
+
+      def assert_valid_path!(path)
+        enclosing_directory = File.dirname(path)
+        unless File.directory?(enclosing_directory)
+          raise InvalidLogDevice, "You must create the enclosing directory #{enclosing_directory} before the log file #{path} can be created."
+        end
+        if File.exist?(path)
+          unless File.writable?(path)
+            raise InvalidLogDevice, "The log file you specified (#{path}) is not writable by user #{Process.euid}"
+          end
+        elsif !File.writable?(enclosing_directory)
+          raise InvalidLogDevice, "You specified a log file #{path} but user #{Process.euid} is not permitted to create files there."
+        end
+      end
+
+    end
+  end
+end

data/lib/chef/expander/node.rb

@@ -0,0 +1,177 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'uuidtools'
+require 'amqp'
+require 'mq'
+require 'open3'
+
+require 'chef/expander/loggable'
+
+module Chef
+  module Expander
+    class Node
+
+      include Loggable
+
+      def self.from_hash(node_info)
+        new(node_info[:guid], node_info[:hostname_f], node_info[:pid])
+      end
+
+      def self.local_node
+        new(guid, hostname_f, Process.pid)
+      end
+
+      def self.guid
+        return @guid if @guid
+        @guid = UUIDTools::UUID.random_create.to_s
+      end
+
+      def self.hostname_f
+        @hostname ||= Open3.popen3("hostname -f") {|stdin, stdout, stderr| stdout.read }.strip
+      end
+
+      attr_reader :guid
+
+      attr_reader :hostname_f
+
+      attr_reader :pid
+
+      def initialize(guid, hostname_f, pid)
+        @guid, @hostname_f, @pid = guid, hostname_f, pid
+      end
+
+      def start(&message_handler)
+        attach_to_queue(exclusive_control_queue, "exclusive control", &message_handler)
+        attach_to_queue(shared_control_queue, "shared_control", &message_handler)
+        attach_to_queue(broadcast_control_queue, "broadcast control", &message_handler)
+      end
+
+      def attach_to_queue(queue, colloquial_name, &message_handler)
+        queue.subscribe(:ack => true) do |headers, payload|
+          log.debug { "received message on #{colloquial_name} queue: #{payload}" }
+          message_handler.call(payload)
+          headers.ack
+        end
+      end
+
+      def stop
+        log.debug { "unsubscribing from broadcast control queue"}
+        broadcast_control_queue.unsubscribe(:nowait => false)
+
+        log.debug { "unsubscribing from shared control queue" }
+        shared_control_queue.unsubscribe(:nowait => false)
+
+        log.debug { "unsubscribing from exclusive control queue" }
+        exclusive_control_queue.unsubscribe(:nowait => false)
+      end
+
+      def direct_message(message)
+        log.debug { "publishing direct message to node #{identifier}: #{message}" }
+        exclusive_control_queue.publish(message)
+      end
+
+      def shared_message(message)
+        log.debug { "publishing shared message #{message}"}
+        shared_control_queue.publish(message)
+      end
+
+      def broadcast_message(message)
+        log.debug { "publishing broadcast message #{message}" }
+        broadcast_control_exchange.publish(message)
+      end
+
+      # The exclusive control queue is for point-to-point messaging, i.e.,
+      # messages directly addressed to this node
+      def exclusive_control_queue
+        @exclusive_control_queue ||= begin
+          log.debug { "declaring exclusive control queue #{exclusive_control_queue_name}" }
+          MQ.queue(exclusive_control_queue_name)
+        end
+      end
+
+      # The shared control queue is for 1 to (1 of N) messaging, i.e.,
+      # messages that can go to any one node.
+      def shared_control_queue
+        @shared_control_queue ||= begin
+          log.debug { "declaring shared control queue #{shared_control_queue_name}" }
+          MQ.queue(shared_control_queue_name)
+        end
+      end
+
+      # The broadcast control queue is for 1 to N messaging, i.e.,
+      # messages that go to every node
+      def broadcast_control_queue
+        @broadcast_control_queue ||= begin
+          log.debug { "declaring broadcast control queue #{broadcast_control_queue_name}"}
+          q = MQ.queue(broadcast_control_queue_name)
+          log.debug { "binding broadcast control queue to broadcast control exchange"}
+          q.bind(broadcast_control_exchange)
+          q
+        end
+      end
+
+      def broadcast_control_exchange
+        @broadcast_control_exchange ||= begin
+          log.debug { "declaring broadcast control exchange opscode-platfrom-control--broadcast" }
+          MQ.fanout(broadcast_control_exchange_name, :nowait => false)
+        end
+      end
+
+      def shared_control_queue_name
+        SHARED_CONTROL_QUEUE_NAME
+      end
+
+      def broadcast_control_queue_name
+        @broadcast_control_queue_name ||= "#{identifier}--broadcast"
+      end
+
+      def broadcast_control_exchange_name
+        BROADCAST_CONTROL_EXCHANGE_NAME
+      end
+
+      def exclusive_control_queue_name
+        @exclusive_control_queue_name ||= "#{identifier}--exclusive-control"
+      end
+
+      def identifier
+        "#{hostname_f}--#{pid}--#{guid}"
+      end
+
+      def ==(other)
+        other.respond_to?(:guid) && other.respond_to?(:hostname_f) && other.respond_to?(:pid) &&
+        (other.guid == guid) && (other.hostname_f == hostname_f) && (other.pid == pid)
+      end
+
+      def eql?(other)
+        (other.class == self.class) && (other.hash == hash)
+      end
+
+      def hash
+        identifier.hash
+      end
+
+      def to_hash
+        {:guid => @guid, :hostname_f => @hostname_f, :pid => @pid}
+      end
+
+    end
+  end
+end

data/lib/chef/expander/solrizer.rb

@@ -0,0 +1,275 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'set'
+require 'yajl'
+require 'fast_xs'
+require 'em-http-request'
+require 'chef/expander/loggable'
+require 'chef/expander/flattener'
+
+module Chef
+  module Expander
+    class Solrizer
+
+      @active_http_requests = Set.new
+
+      def self.http_request_started(instance)
+        @active_http_requests << instance
+      end
+
+      def self.http_request_completed(instance)
+        @active_http_requests.delete(instance)
+      end
+
+      def self.http_requests_active?
+        !@active_http_requests.empty?
+      end
+
+      def self.clear_http_requests
+        @active_http_requests.clear
+      end
+
+      include Loggable
+
+      ADD     = "add"
+      DELETE  = "delete"
+      SKIP    = "skip"
+
+      ITEM        = "item"
+      ID          = "id"
+      TYPE        = "type"
+      DATABASE    = "database"
+      ENQUEUED_AT = "enqueued_at"
+
+      DATA_BAG_ITEM = "data_bag_item"
+      DATA_BAG      = "data_bag"
+
+      X_CHEF_id_CHEF_X        = 'X_CHEF_id_CHEF_X'
+      X_CHEF_database_CHEF_X  = 'X_CHEF_database_CHEF_X'
+      X_CHEF_type_CHEF_X      = 'X_CHEF_type_CHEF_X'
+
+      CONTENT_TYPE_XML = {"Content-Type" => "text/xml"}
+
+      attr_reader :action
+
+      attr_reader :indexer_payload
+
+      attr_reader :chef_object
+
+      attr_reader :obj_id
+
+      attr_reader :obj_type
+
+      attr_reader :database
+
+      attr_reader :enqueued_at
+
+      def initialize(object_command_json, &on_completion_block)
+        @start_time = Time.now.to_f
+        @on_completion_block = on_completion_block
+        if parsed_message    = parse(object_command_json)
+          @action           = parsed_message["action"]
+          @indexer_payload  = parsed_message["payload"]
+
+          extract_object_fields if @indexer_payload
+        else
+          @action = SKIP
+        end
+      end
+
+      def extract_object_fields
+        @chef_object = @indexer_payload[ITEM]
+        @database    = @indexer_payload[DATABASE]
+        @obj_id      = @indexer_payload[ID]
+        @obj_type    = @indexer_payload[TYPE]
+        @enqueued_at = @indexer_payload[ENQUEUED_AT]
+        @data_bag = @obj_type == DATA_BAG_ITEM ? @chef_object[DATA_BAG] : nil
+      end
+
+      def parse(serialized_object)
+        Yajl::Parser.parse(serialized_object)
+      rescue Yajl::ParseError
+        log.error { "cannot index object because it is invalid JSON: #{serialized_object}" }
+      end
+
+      def run
+        case @action
+        when ADD
+          add
+        when DELETE
+          delete
+        when SKIP
+          completed
+          log.info { "not indexing this item because of malformed JSON"}
+        else
+          completed
+          log.error { "cannot index object becuase it has an invalid action #{@action}" }
+        end
+      end
+
+      def add
+        post_to_solr(pointyize_add) do
+          ["indexed #{indexed_object}",
+           "transit,xml,solr-post |",
+           [transit_time, @xml_time, @solr_post_time].join(","),
+           "|"
+          ].join(" ")
+        end
+      rescue Exception => e
+        log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}"}
+      end
+
+      def delete
+        post_to_solr(pointyize_delete) { "deleted #{indexed_object} transit-time[#{transit_time}s]"}
+      rescue Exception => e
+        log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}"}
+      end
+
+      def flattened_object
+        flattened_object = Flattener.new(@chef_object).flattened_item
+ 
+        flattened_object[X_CHEF_id_CHEF_X]        = [@obj_id]
+        flattened_object[X_CHEF_database_CHEF_X]  = [@database]
+        flattened_object[X_CHEF_type_CHEF_X]      = [@obj_type]
+
+        log.debug {"adding flattened object to Solr: #{flattened_object.inspect}"}
+
+        flattened_object
+      end
+
+      START_XML   = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+      ADD_DOC     = "<add><doc>"
+      DELETE_DOC  = "<delete>"
+      ID_OPEN     = "<id>"
+      ID_CLOSE    = "</id>"
+      END_ADD_DOC = "</doc></add>\n"
+      END_DELETE  = "</delete>\n"
+      START_CONTENT = '<field name="content">'
+      CLOSE_FIELD = "</field>"
+
+      FLD_CHEF_ID_FMT = '<field name="X_CHEF_id_CHEF_X">%s</field>'
+      FLD_CHEF_DB_FMT = '<field name="X_CHEF_database_CHEF_X">%s</field>'
+      FLD_CHEF_TY_FMT = '<field name="X_CHEF_type_CHEF_X">%s</field>'
+      FLD_DATA_BAG    = '<field name="data_bag">%s</field>'
+
+      KEYVAL_FMT = "%s__=__%s "
+
+      # Takes a flattened hash where the values are arrays and converts it into
+      # a dignified XML document suitable for POST to Solr.
+      # The general structure of the output document is like this:
+      #   <?xml version="1.0" encoding="UTF-8"?>
+      #   <add>
+      #     <doc>
+      #       <field name="content">
+      #           key__=__value
+      #           key__=__another_value
+      #           other_key__=__yet another value
+      #       </field>
+      #     </doc>
+      #   </add>
+      # The document as generated has minimal newlines and formatting, however.
+      def pointyize_add
+        xml = ""
+        xml << START_XML << ADD_DOC
+        xml << (FLD_CHEF_ID_FMT % @obj_id)
+        xml << (FLD_CHEF_DB_FMT % @database)
+        xml << (FLD_CHEF_TY_FMT % @obj_type)
+        xml << START_CONTENT
+        content = ""
+        flattened_object.each do |field, values|
+          values.each do |v|
+            content << (KEYVAL_FMT % [field, v])
+          end
+        end
+        xml << content.fast_xs
+        xml << CLOSE_FIELD      # ends content
+        xml << (FLD_DATA_BAG % @data_bag.fast_xs) if @data_bag
+        xml << END_ADD_DOC
+        @xml_time = Time.now.to_f - @start_time
+        xml
+      end
+
+      # Takes a succinct document id, like 2342, and turns it into something
+      # even more compact, like
+      #   "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><id>2342</id></delete>\n"
+      def pointyize_delete
+        xml = ""
+        xml << START_XML
+        xml << DELETE_DOC
+        xml << ID_OPEN
+        xml << @obj_id.to_s
+        xml << ID_CLOSE
+        xml << END_DELETE
+        xml
+      end
+
+      def post_to_solr(document, &logger_block)
+        log.debug("POSTing document to SOLR:\n#{document}")
+        http_req = EventMachine::HttpRequest.new(solr_url).post(:body => document, :timeout => 1200, :head => CONTENT_TYPE_XML)
+        http_request_started
+
+        http_req.callback do
+          completed
+          if http_req.response_header.status == 200
+            log.info(&logger_block)
+          else
+            log.error { "Failed to post to solr: #{indexed_object}" }
+          end
+        end
+        http_req.errback do
+          completed
+          log.error { "Failed to post to solr (connection error): #{indexed_object}" }
+        end
+      end
+
+      def completed
+        @solr_post_time = Time.now.to_f - @start_time
+        self.class.http_request_completed(self)
+        @on_completion_block.call
+      end
+
+      def transit_time
+        Time.now.utc.to_i - @enqueued_at
+      end
+
+      def solr_url
+        'http://127.0.0.1:8983/solr/update'
+      end
+
+      def indexed_object
+        "#{@obj_type}[#{@obj_id}] database[#{@database}]"
+      end
+
+      def http_request_started
+        self.class.http_request_started(self)
+      end
+
+      def eql?(other)
+        other.hash == hash
+      end
+
+      def hash
+        "#{action}#{indexed_object}#@enqueued_at#{self.class.name}".hash
+      end
+
+    end
+  end
+end