erchef-expander 11.4.0

data/lib/chef/expander/solrizer.rb

@@ -0,0 +1,275 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'set'
+require 'yajl'
+require 'fast_xs'
+require 'em-http-request'
+require 'chef/expander/loggable'
+require 'chef/expander/flattener'
+
+module Chef
+  module Expander
+    class Solrizer
+
+      @active_http_requests = Set.new
+
+      def self.http_request_started(instance)
+        @active_http_requests << instance
+      end
+
+      def self.http_request_completed(instance)
+        @active_http_requests.delete(instance)
+      end
+
+      def self.http_requests_active?
+        !@active_http_requests.empty?
+      end
+
+      def self.clear_http_requests
+        @active_http_requests.clear
+      end
+
+      include Loggable
+
+      ADD     = "add"
+      DELETE  = "delete"
+      SKIP    = "skip"
+
+      ITEM        = "item"
+      ID          = "id"
+      TYPE        = "type"
+      DATABASE    = "database"
+      ENQUEUED_AT = "enqueued_at"
+
+      DATA_BAG_ITEM = "data_bag_item"
+      DATA_BAG      = "data_bag"
+
+      X_CHEF_id_CHEF_X        = 'X_CHEF_id_CHEF_X'
+      X_CHEF_database_CHEF_X  = 'X_CHEF_database_CHEF_X'
+      X_CHEF_type_CHEF_X      = 'X_CHEF_type_CHEF_X'
+
+      CONTENT_TYPE_XML = {"Content-Type" => "text/xml"}
+
+      attr_reader :action
+
+      attr_reader :indexer_payload
+
+      attr_reader :chef_object
+
+      attr_reader :obj_id
+
+      attr_reader :obj_type
+
+      attr_reader :database
+
+      attr_reader :enqueued_at
+
+      def initialize(object_command_json, &on_completion_block)
+        @start_time = Time.now.to_f
+        @on_completion_block = on_completion_block
+        if parsed_message    = parse(object_command_json)
+          @action           = parsed_message["action"]
+          @indexer_payload  = parsed_message["payload"]
+
+          extract_object_fields if @indexer_payload
+        else
+          @action = SKIP
+        end
+      end
+
+      def extract_object_fields
+        @chef_object = @indexer_payload[ITEM]
+        @database    = @indexer_payload[DATABASE]
+        @obj_id      = @indexer_payload[ID]
+        @obj_type    = @indexer_payload[TYPE]
+        @enqueued_at = @indexer_payload[ENQUEUED_AT]
+        @data_bag = @obj_type == DATA_BAG_ITEM ? @chef_object[DATA_BAG] : nil
+      end
+
+      def parse(serialized_object)
+        Yajl::Parser.parse(serialized_object)
+      rescue Yajl::ParseError
+        log.error { "cannot index object because it is invalid JSON: #{serialized_object}" }
+      end
+
+      def run
+        case @action
+        when ADD
+          add
+        when DELETE
+          delete
+        when SKIP
+          completed
+          log.info { "not indexing this item because of malformed JSON"}
+        else
+          completed
+          log.error { "cannot index object becuase it has an invalid action #{@action}" }
+        end
+      end
+
+      def add
+        post_to_solr(pointyize_add) do
+          ["indexed #{indexed_object}",
+           "transit,xml,solr-post |",
+           [transit_time, @xml_time, @solr_post_time].join(","),
+           "|"
+          ].join(" ")
+        end
+      rescue Exception => e
+        log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}"}
+      end
+
+      def delete
+        post_to_solr(pointyize_delete) { "deleted #{indexed_object} transit-time[#{transit_time}s]"}
+      rescue Exception => e
+        log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}"}
+      end
+
+      def flattened_object
+        flattened_object = Flattener.new(@chef_object).flattened_item
+ 
+        flattened_object[X_CHEF_id_CHEF_X]        = [@obj_id]
+        flattened_object[X_CHEF_database_CHEF_X]  = [@database]
+        flattened_object[X_CHEF_type_CHEF_X]      = [@obj_type]
+
+        log.debug {"adding flattened object to Solr: #{flattened_object.inspect}"}
+
+        flattened_object
+      end
+
+      START_XML   = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+      ADD_DOC     = "<add><doc>"
+      DELETE_DOC  = "<delete>"
+      ID_OPEN     = "<id>"
+      ID_CLOSE    = "</id>"
+      END_ADD_DOC = "</doc></add>\n"
+      END_DELETE  = "</delete>\n"
+      START_CONTENT = '<field name="content">'
+      CLOSE_FIELD = "</field>"
+
+      FLD_CHEF_ID_FMT = '<field name="X_CHEF_id_CHEF_X">%s</field>'
+      FLD_CHEF_DB_FMT = '<field name="X_CHEF_database_CHEF_X">%s</field>'
+      FLD_CHEF_TY_FMT = '<field name="X_CHEF_type_CHEF_X">%s</field>'
+      FLD_DATA_BAG    = '<field name="data_bag">%s</field>'
+
+      KEYVAL_FMT = "%s__=__%s "
+
+      # Takes a flattened hash where the values are arrays and converts it into
+      # a dignified XML document suitable for POST to Solr.
+      # The general structure of the output document is like this:
+      #   <?xml version="1.0" encoding="UTF-8"?>
+      #   <add>
+      #     <doc>
+      #       <field name="content">
+      #           key__=__value
+      #           key__=__another_value
+      #           other_key__=__yet another value
+      #       </field>
+      #     </doc>
+      #   </add>
+      # The document as generated has minimal newlines and formatting, however.
+      def pointyize_add
+        xml = ""
+        xml << START_XML << ADD_DOC
+        xml << (FLD_CHEF_ID_FMT % @obj_id)
+        xml << (FLD_CHEF_DB_FMT % @database)
+        xml << (FLD_CHEF_TY_FMT % @obj_type)
+        xml << START_CONTENT
+        content = ""
+        flattened_object.each do |field, values|
+          values.each do |v|
+            content << (KEYVAL_FMT % [field, v])
+          end
+        end
+        xml << content.fast_xs
+        xml << CLOSE_FIELD      # ends content
+        xml << (FLD_DATA_BAG % @data_bag.fast_xs) if @data_bag
+        xml << END_ADD_DOC
+        @xml_time = Time.now.to_f - @start_time
+        xml
+      end
+
+      # Takes a succinct document id, like 2342, and turns it into something
+      # even more compact, like
+      #   "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><id>2342</id></delete>\n"
+      def pointyize_delete
+        xml = ""
+        xml << START_XML
+        xml << DELETE_DOC
+        xml << ID_OPEN
+        xml << @obj_id.to_s
+        xml << ID_CLOSE
+        xml << END_DELETE
+        xml
+      end
+
+      def post_to_solr(document, &logger_block)
+        log.debug("POSTing document to SOLR:\n#{document}")
+        http_req = EventMachine::HttpRequest.new(solr_url).post(:body => document, :timeout => 1200, :head => CONTENT_TYPE_XML)
+        http_request_started
+
+        http_req.callback do
+          completed
+          if http_req.response_header.status == 200
+            log.info(&logger_block)
+          else
+            log.error { "Failed to post to solr: #{indexed_object}" }
+          end
+        end
+        http_req.errback do
+          completed
+          log.error { "Failed to post to solr (connection error): #{indexed_object}" }
+        end
+      end
+
+      def completed
+        @solr_post_time = Time.now.to_f - @start_time
+        self.class.http_request_completed(self)
+        @on_completion_block.call
+      end
+
+      def transit_time
+        Time.now.utc.to_i - @enqueued_at
+      end
+
+      def solr_url
+        "#{Expander.config.solr_url}/update"
+      end
+
+      def indexed_object
+        "#{@obj_type}[#{@obj_id}] database[#{@database}]"
+      end
+
+      def http_request_started
+        self.class.http_request_started(self)
+      end
+
+      def eql?(other)
+        other.hash == hash
+      end
+
+      def hash
+        "#{action}#{indexed_object}#@enqueued_at#{self.class.name}".hash
+      end
+
+    end
+  end
+end

data/lib/chef/expander/version.rb

@@ -0,0 +1,41 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'open3'
+
+module Chef
+  module Expander
+
+    VERSION = "11.0.0.alpha.1"
+
+    def self.version
+      @rev ||= begin
+        begin
+          rev = Open3.popen3("git rev-parse HEAD") {|stdin, stdout, stderr| stdout.read }.strip
+        rescue Errno::ENOENT
+          rev = ""
+        end
+        rev.empty? ? nil : " (#{rev})"
+      end
+      "#{VERSION}#@rev"
+    end
+
+  end
+end

data/lib/chef/expander/vnode.rb

@@ -0,0 +1,106 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'eventmachine'
+require 'amqp'
+require 'mq'
+
+require 'chef/expander/loggable'
+require 'chef/expander/solrizer'
+
+module Chef
+  module Expander
+    class VNode
+      include Loggable
+
+      attr_reader :vnode_number
+
+      attr_reader :supervise_interval
+
+      def initialize(vnode_number, supervisor, opts={})
+        @vnode_number = vnode_number.to_i
+        @supervisor   = supervisor
+        @queue    = nil
+        @stopped  = false
+        @supervise_interval = opts[:supervise_interval] || 30
+      end
+
+      def start
+        @supervisor.vnode_added(self)
+
+        subscription_confirmed = Proc.new do
+          abort_on_multiple_subscribe
+          supervise_consumer_count
+        end
+
+        queue.subscribe(:ack => true, :confirm => subscription_confirmed) do |headers, payload|
+          log.debug {"got #{payload} size(#{payload.size} bytes) on queue #{queue_name}"}
+          solrizer = Solrizer.new(payload) { headers.ack }
+          solrizer.run
+        end
+
+      rescue MQ::Error => e
+        log.error {"Failed to start subscriber on #{queue_name} #{e.class.name}: #{e.message}"}
+      end
+
+      def supervise_consumer_count
+        EM.add_periodic_timer(supervise_interval) do
+          abort_on_multiple_subscribe
+        end
+      end
+
+      def abort_on_multiple_subscribe
+        queue.status do |message_count, subscriber_count|
+          if subscriber_count.to_i > 1
+            log.error { "Detected extra consumers (#{subscriber_count} total) on queue #{queue_name}, cancelling subscription" }
+            stop
+          end
+        end
+      end
+
+      def stop
+        log.debug {"Cancelling subscription on queue #{queue_name.inspect}"}
+        queue.unsubscribe if queue.subscribed?
+        @supervisor.vnode_removed(self)
+        @stopped = true
+      end
+
+      def stopped?
+        @stopped
+      end
+
+      def queue
+        @queue ||= begin
+          log.debug { "declaring queue #{queue_name}" }
+          MQ.queue(queue_name, :passive => false, :durable => true)
+        end
+      end
+
+      def queue_name
+        "vnode-#{@vnode_number}"
+      end
+
+      def control_queue_name
+        "#{queue_name}-control"
+      end
+
+    end
+  end
+end

data/lib/chef/expander/vnode_supervisor.rb

@@ -0,0 +1,265 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'yajl'
+require 'eventmachine'
+require 'amqp'
+require 'mq'
+require 'chef/expander/version'
+require 'chef/expander/loggable'
+require 'chef/expander/node'
+require 'chef/expander/vnode'
+require 'chef/expander/vnode_table'
+require 'chef/expander/configuration'
+
+module ::AMQP
+  def self.hard_reset!
+    MQ.reset rescue nil
+    stop
+    EM.stop rescue nil
+    Thread.current[:mq], @conn = nil, nil
+  end
+end
+
+module Chef
+  module Expander
+    class VNodeSupervisor
+      include Loggable
+      extend  Loggable
+
+      COULD_NOT_CONNECT = /Could not connect to server/.freeze
+
+      def self.start_cluster_worker
+        @vnode_supervisor = new
+        @original_ppid = Process.ppid
+        trap_signals
+
+        vnodes = Expander.config.vnode_numbers
+
+        $0 = "chef-expander#{Expander.config.ps_tag} worker ##{Expander.config.index} (vnodes #{vnodes.min}-#{vnodes.max})"
+        
+        AMQP.start(Expander.config.amqp_config) do
+          start_consumers
+          await_parent_death
+        end
+      end
+
+      def self.await_parent_death
+        @awaiting_parent_death = EM.add_periodic_timer(1) do
+          unless Process.ppid == @original_ppid
+            @awaiting_parent_death.cancel
+            stop_immediately("master process death")
+          end
+        end
+      end
+
+      def self.start
+        @vnode_supervisor = new
+        trap_signals
+
+        Expander.init_config(ARGV)
+
+        log.info("Chef Search Expander #{Expander.version} starting up.")
+
+        begin
+          AMQP.start(Expander.config.amqp_config) do
+            start_consumers
+          end
+        rescue AMQP::Error => e
+          if e.message =~ COULD_NOT_CONNECT
+            log.error { "Could not connect to rabbitmq. Make sure it is running and correctly configured." }
+            log.error { e.message }
+
+            AMQP.hard_reset!
+
+            sleep 5
+            retry
+          else
+            raise
+          end
+        end
+      end
+
+      def self.start_consumers
+        log.debug { "Setting prefetch count to 1"}
+        MQ.prefetch(1)
+
+        vnodes = Expander.config.vnode_numbers
+        log.info("Starting Consumers for vnodes #{vnodes.min}-#{vnodes.max}")
+        @vnode_supervisor.start(vnodes)
+      end
+
+      def self.trap_signals
+        Kernel.trap(:INT)  { stop_immediately(:INT) }
+        Kernel.trap(:TERM) { stop_gracefully(:TERM) }
+      end
+
+      def self.stop_immediately(signal)
+        log.info { "Initiating immediate shutdown on signal (#{signal})" }
+        @vnode_supervisor.stop
+        EM.add_timer(1) do
+          AMQP.stop
+          EM.stop
+        end
+      end
+
+      def self.stop_gracefully(signal)
+        log.info { "Initiating graceful shutdown on signal (#{signal})" }
+        @vnode_supervisor.stop
+        wait_for_http_requests_to_complete
+      end
+
+      def self.wait_for_http_requests_to_complete
+        if Expander::Solrizer.http_requests_active?
+          log.info { "waiting for in progress HTTP Requests to complete"}
+          EM.add_timer(1) do
+            wait_for_http_requests_to_complete
+          end
+        else
+          log.info { "HTTP requests completed, shutting down"}
+          AMQP.stop
+          EM.stop
+        end
+      end
+
+      attr_reader :vnode_table
+
+      attr_reader :local_node
+
+      def initialize
+        @vnodes = {}
+        @vnode_table = VNodeTable.new(self)
+        @local_node  = Node.local_node
+        @queue_name, @guid = nil, nil
+      end
+
+      def start(vnode_ids)
+        @local_node.start do |message|
+          process_control_message(message)
+        end
+
+        #start_vnode_table_publisher
+
+        Array(vnode_ids).each { |vnode_id| spawn_vnode(vnode_id) }
+      end
+
+      def stop
+        @local_node.stop
+
+        #log.debug { "stopping vnode table updater" }
+        #@vnode_table_publisher.cancel
+
+        log.info { "Stopping VNode queue subscribers"}
+        @vnodes.each do |vnode_number, vnode|
+          log.debug { "Stopping consumer on VNode #{vnode_number}"}
+          vnode.stop
+        end
+        
+      end
+
+      def vnode_added(vnode)
+        log.debug { "vnode #{vnode.vnode_number} registered with supervisor" }
+        @vnodes[vnode.vnode_number.to_i] = vnode
+      end
+
+      def vnode_removed(vnode)
+        log.debug { "vnode #{vnode.vnode_number} unregistered from supervisor" }
+        @vnodes.delete(vnode.vnode_number.to_i)
+      end
+
+      def vnodes
+        @vnodes.keys.sort
+      end
+
+      def spawn_vnode(vnode_number)
+        VNode.new(vnode_number, self).start
+      end
+
+      def release_vnode
+        # TODO
+      end
+
+      def process_control_message(message)
+        control_message = parse_symbolic(message)
+        case control_message[:action]
+        when "claim_vnode"
+          spawn_vnode(control_message[:vnode_id])
+        when "recover_vnode"
+          recover_vnode(control_message[:vnode_id])
+        when "release_vnodes"
+          raise "todo"
+          release_vnode()
+        when "update_vnode_table"
+          @vnode_table.update_table(control_message[:data])
+        when "vnode_table_publish"
+          publish_vnode_table
+        when "status"
+          publish_status_to(control_message[:rsvp])
+        when "set_log_level"
+          set_log_level(control_message[:level], control_message[:rsvp])
+        else
+          log.error { "invalid control message #{control_message.inspect}" }
+        end
+      rescue Exception => e
+        log.error { "Error processing a control message."}
+        log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}" }
+      end
+
+
+      def start_vnode_table_publisher
+        @vnode_table_publisher = EM.add_periodic_timer(10) { publish_vnode_table }
+      end
+
+      def publish_vnode_table
+        status_update = @local_node.to_hash
+        status_update[:vnodes] = vnodes
+        status_update[:update] = :add
+        @local_node.broadcast_message(Yajl::Encoder.encode({:action => :update_vnode_table, :data => status_update}))
+      end
+
+      def publish_status_to(return_queue)
+        status_update = @local_node.to_hash
+        status_update[:vnodes] = vnodes
+        MQ.queue(return_queue).publish(Yajl::Encoder.encode(status_update))
+      end
+
+      def set_log_level(level, rsvp_to)
+        log.info { "setting log level to #{level} due to command from #{rsvp_to}" }
+        new_log_level = (Expander.config.log_level = level.to_sym)
+        reply = {:level => new_log_level, :node => @local_node.to_hash}
+        MQ.queue(rsvp_to).publish(Yajl::Encoder.encode(reply))
+      end
+
+      def recover_vnode(vnode_id)
+        if @vnode_table.local_node_is_leader?
+          log.debug { "Recovering vnode: #{vnode_id}" }
+          @local_node.shared_message(Yajl::Encoder.encode({:action => :claim_vnode, :vnode_id => vnode_id}))
+        else
+          log.debug { "Ignoring :recover_vnode message because this node is not the leader" }
+        end
+      end
+
+      def parse_symbolic(message)
+        Yajl::Parser.new(:symbolize_keys => true).parse(message)
+      end
+
+    end
+  end
+end