chef-solr 0.8.2

data/lib/chef/solr/application/indexer.rb

@@ -0,0 +1,133 @@
+#
+# Author:: AJ Christensen (<aj@opscode.com)
+# Copyright:: Copyright (c) 2008 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'chef'
+require 'chef/log'
+require 'chef/config'
+require 'chef/application'
+require 'chef/solr'
+require 'chef/solr/index'
+require 'chef/solr/index_queue_consumer'
+require 'chef/daemon'
+require 'chef/webui_user'
+
+class Chef
+  class Solr
+    class Application
+      class Indexer < Chef::Application
+  
+        option :config_file, 
+          :short => "-c CONFIG",
+          :long  => "--config CONFIG",
+          :default => "/etc/chef/solr.rb",
+          :description => "The configuration file to use"
+
+        option :log_level, 
+          :short        => "-l LEVEL",
+          :long         => "--log_level LEVEL",
+          :description  => "Set the log level (debug, info, warn, error, fatal)",
+          :proc         => lambda { |l| l.to_sym }
+
+        option :log_location,
+          :short        => "-L LOGLOCATION",
+          :long         => "--logfile LOGLOCATION",
+          :description  => "Set the log file location, defaults to STDOUT - recommended for daemonizing",
+          :proc         => nil
+
+        option :help,
+          :short        => "-h",
+          :long         => "--help",
+          :description  => "Show this message",
+          :on           => :tail,
+          :boolean      => true,
+          :show_options => true,
+          :exit         => 0
+
+        option :user,
+          :short => "-u USER",
+          :long => "--user USER",
+          :description => "User to set privilege to",
+          :proc => nil
+
+        option :group,
+          :short => "-g GROUP",
+          :long => "--group GROUP",
+          :description => "Group to set privilege to",
+          :proc => nil
+
+        option :daemonize,
+          :short => "-d",
+          :long => "--daemonize",
+          :description => "Daemonize the process",
+          :proc => lambda { |p| true }
+
+        option :amqp_host,
+          :long => "--amqp-host HOST",
+          :description => "The amqp host"
+
+        option :amqp_port,
+          :long => "--amqp-port PORT",
+          :description => "The amqp port"
+
+        option :amqp_user,
+          :long => "--amqp-user USER",
+          :description => "The amqp user"
+
+        option :amqp_pass,
+          :long => "--amqp-pass PASS",
+          :description => "The amqp password"
+
+        option :amqp_vhost,
+          :long => "--amqp-vhost VHOST",
+          :description => "The amqp vhost"
+        
+        Signal.trap("INT") do
+          begin
+            AmqpClient.instance.stop
+          rescue Bunny::ProtocolError, Bunny::ConnectionError, Bunny::UnsubscribeError
+          end
+          fatal!("SIGINT received, stopping", 2)
+        end
+        
+        Kernel.trap("TERM") do 
+          begin
+            AmqpClient.instance.stop
+          rescue Bunny::ProtocolError, Bunny::ConnectionError, Bunny::UnsubscribeError
+          end
+          fatal!("SIGTERM received, stopping", 1)
+        end
+        
+        def initialize
+          super
+
+          @index = Chef::Solr::Index.new
+          @consumer = Chef::Solr::IndexQueueConsumer.new
+        end
+
+        def setup_application
+          Chef::Daemon.change_privilege
+          Chef::Log.level = Chef::Config[:log_level]
+        end
+
+        def run_application
+          Chef::Daemon.daemonize("chef-solr-indexer") if Chef::Config[:daemonize]
+          @consumer.start
+        end
+      end
+    end
+  end
+end

data/lib/chef/solr/application/rebuild.rb

@@ -0,0 +1,93 @@
+#
+# Author:: AJ Christensen (<aj@opscode.com)
+# Copyright:: Copyright (c) 2008 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'chef'
+require 'chef/log'
+require 'chef/config'
+require 'chef/application'
+require 'chef/solr'
+require 'chef/solr/index'
+
+class Chef
+  class Solr
+    class Application
+      class Rebuild < Chef::Application
+  
+        option :config_file, 
+          :short => "-c CONFIG",
+          :long  => "--config CONFIG",
+          :default => "/etc/chef/solr.rb",
+          :description => "The configuration file to use"
+
+        option :log_level, 
+          :short        => "-l LEVEL",
+          :long         => "--log_level LEVEL",
+          :description  => "Set the log level (debug, info, warn, error, fatal)",
+          :proc         => lambda { |l| l.to_sym }
+
+        option :log_location,
+          :short        => "-L LOGLOCATION",
+          :long         => "--logfile LOGLOCATION",
+          :description  => "Set the log file location, defaults to STDOUT - recommended for daemonizing",
+          :proc         => nil
+
+        option :help,
+          :short        => "-h",
+          :long         => "--help",
+          :description  => "Show this message",
+          :on           => :tail,
+          :boolean      => true,
+          :show_options => true,
+          :exit         => 0
+
+        option :couchdb_database,
+          :short => "-d DB",
+          :long => "--couchdb-database DB",
+          :description => "The CouchDB Database to re-index"
+
+        option :couchdb_url,
+          :short => "-u URL",
+          :long => "--couchdb-url URL",
+          :description => "The CouchDB URL"
+
+        def initialize
+          super
+
+          @index = Chef::Solr::Index.new
+        end
+
+        def setup_application
+          Chef::Log.level = Chef::Config[:log_level]
+          Chef::Log.warn("This operation is destructive!")
+          Chef::Log.warn("I'm going to count to 10, and then delete your Solr index and rebuild it.")
+          Chef::Log.warn("CTRL-C will, of course, stop this disaster.")
+          0.upto(10) do |num|
+            Chef::Log.warn("... #{num}")
+            sleep 1
+          end
+          Chef::Log.warn("... Bombs away!")
+        end
+
+        def run_application
+          s = Chef::Solr.new(Chef::Config[:solr_url])
+          Chef::Log.info("Destroying the index")
+          s.rebuild_index
+        end
+      end
+    end
+  end
+end

data/lib/chef/solr/application/solr.rb

@@ -0,0 +1,165 @@
+#
+# Author:: AJ Christensen (<aj@opscode.com)
+# Copyright:: Copyright (c) 2008 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'chef'
+require 'chef/log'
+require 'chef/config'
+require 'chef/application'
+require 'chef/daemon'
+require 'chef/client'
+
+class Chef
+  class Solr
+    class Application
+      class Solr < Chef::Application
+  
+        option :config_file, 
+          :short => "-c CONFIG",
+          :long  => "--config CONFIG",
+          :default => "/etc/chef/solr.rb",
+          :description => "The configuration file to use"
+
+        option :log_level, 
+          :short        => "-l LEVEL",
+          :long         => "--log_level LEVEL",
+          :description  => "Set the log level (debug, info, warn, error, fatal)",
+          :proc         => lambda { |l| l.to_sym }
+
+        option :log_location,
+          :short        => "-L LOGLOCATION",
+          :long         => "--logfile LOGLOCATION",
+          :description  => "Set the log file location, defaults to STDOUT - recommended for daemonizing",
+          :proc         => nil
+
+        option :help,
+          :short        => "-h",
+          :long         => "--help",
+          :description  => "Show this message",
+          :on           => :tail,
+          :boolean      => true,
+          :show_options => true,
+          :exit         => 0
+
+        option :user,
+          :short => "-u USER",
+          :long => "--user USER",
+          :description => "User to set privilege to",
+          :proc => nil
+
+        option :group,
+          :short => "-g GROUP",
+          :long => "--group GROUP",
+          :description => "Group to set privilege to",
+          :proc => nil
+
+        option :daemonize,
+          :short => "-d",
+          :long => "--daemonize",
+          :description => "Daemonize the process",
+          :proc => lambda { |p| true }
+
+        option :solr_jetty_path,
+          :short => "-W PATH",
+          :long => "--solr-jetty-dir PATH",
+          :description => "Where to place the Solr Jetty instance"
+
+        option :solr_data_path,
+          :short => "-D PATH",
+          :long => "--solr-data-dir PATH",
+          :description => "Where the Solr data lives"
+
+        option :solr_home_path,
+          :short => "-H PATH",
+          :long => "--solr-home-dir PATH",
+          :description => "Solr home directory"
+
+        option :solr_heap_size,
+          :short => "-x SIZE",
+          :long => "--solor-heap-size SIZE",
+          :description => "Set the size of the Java Heap"
+
+        option :solr_java_opts,
+          :short => "-j OPTS",
+          :long => "--java-opts OPTS",
+          :description => "Raw options passed to Java" 
+
+        def initialize
+          super
+          Chef::Log.level = Chef::Config[:log_level] 
+        end
+
+        def setup_application
+          Chef::Daemon.change_privilege
+
+          # Build up a client
+          c = Chef::Client.new
+          c.build_node(nil, true)
+
+          solr_base = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "..", "..", "solr"))
+
+          # Create the Jetty container
+          unless File.directory?(Chef::Config[:solr_jetty_path])
+            Chef::Log.warn("Initializing the Jetty container") 
+            solr_jetty_dir = Chef::Resource::Directory.new(Chef::Config[:solr_jetty_path], nil, c.node)
+            solr_jetty_dir.recursive(true)
+            solr_jetty_dir.run_action(:create)
+            solr_jetty_untar = Chef::Resource::Execute.new("untar_jetty", nil, c.node)
+            solr_jetty_untar.command("tar zxvf #{File.join(solr_base, 'solr-jetty.tar.gz')}")
+            solr_jetty_untar.cwd(Chef::Config[:solr_jetty_path])
+            solr_jetty_untar.run_action(:run)
+          end
+
+          # Create the solr home
+          unless File.directory?(Chef::Config[:solr_home_path])
+            Chef::Log.warn("Initializing Solr home directory") 
+            solr_home_dir = Chef::Resource::Directory.new(Chef::Config[:solr_home_path], nil, c.node)
+            solr_home_dir.recursive(true)
+            solr_home_dir.run_action(:create)
+            solr_jetty_untar = Chef::Resource::Execute.new("untar_solr_home", nil, c.node)
+            solr_jetty_untar.command("tar zxvf #{File.join(solr_base, 'solr-home.tar.gz')}")
+            solr_jetty_untar.cwd(Chef::Config[:solr_home_path])
+            solr_jetty_untar.run_action(:run)
+          end
+
+          # Create the solr data path 
+          unless File.directory?(Chef::Config[:solr_data_path])
+            Chef::Log.warn("Initializing Solr data directory")
+            solr_data_dir = Chef::Resource::Directory.new(Chef::Config[:solr_data_path], nil, c.node)
+            solr_data_dir.recursive(true)
+            solr_data_dir.run_action(:create)
+          end
+        end
+
+        def run_application
+          if Chef::Config[:daemonize]
+            Chef::Daemon.daemonize("chef-solr")
+          end
+          Dir.chdir(Chef::Config[:solr_jetty_path]) do
+            command = "java -Xmx#{Chef::Config[:solr_heap_size]} -Xms#{Chef::Config[:solr_heap_size]}"
+            command << " -Dsolr.data.dir=#{Chef::Config[:solr_data_path]}"
+            command << " -Dsolr.solr.home=#{Chef::Config[:solr_home_path]}"
+            command << " #{Chef::Config[:solr_java_opts]}" if Chef::Config[:solr_java_opts]
+            command << " -jar #{File.join(Chef::Config[:solr_jetty_path], 'start.jar')}"
+            Chef::Log.info("Starting Solr with #{command}")
+            Kernel.exec(command)
+
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chef/solr/index.rb

@@ -0,0 +1,153 @@
+#
+# Author:: Adam Jacob (<adam@opscode.com>)
+# Copyright:: Copyright (c) 2009 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/log'
+require 'chef/config'
+require 'chef/solr'
+require 'libxml'
+require 'net/http'
+
+class Chef
+  class Solr
+    class Index < Solr
+
+      def add(id, database, type, item)
+        raise ArgumentError, "Object must respond to keys!" unless item.respond_to?(:keys)
+        to_index = flatten_and_expand(item)
+        to_index["X_CHEF_id_CHEF_X"] = id
+        to_index["X_CHEF_database_CHEF_X"] = database
+        to_index["X_CHEF_type_CHEF_X"] = type
+        solr_add(to_index)
+        to_index
+      end
+
+      def delete(id)
+        solr_delete_by_id(id)
+      end
+
+      def delete_by_query(query)
+        solr_delete_by_query(query)
+      end
+
+      def flatten_and_expand(item, fields=Hash.new, parent=nil)
+        item.keys.each do |key|
+          # If we have a parent, we want to add the current key as a value 
+          if parent
+            # foo_bar = bar
+            set_field_value(fields, parent, key) 
+            # foo_X = bar, etc.
+            make_expando_fields(parent).each do |ex_key|
+              set_field_value(fields, ex_key, key)
+            end
+          end
+          case item[key]
+          when Hash
+            parent_key = parent ? "#{parent}_#{key}" : key
+            flatten_and_expand(item[key], fields, parent_key)
+          else
+            parent_key = parent ? "#{parent}_#{key}" : key
+            set_field_value(fields, key, item[key])
+            set_field_value(fields, parent_key, item[key]) if parent
+            make_expando_fields(parent_key).each do |ex_key|
+              set_field_value(fields, ex_key, item[key])
+            end
+          end
+        end
+        fields
+      end
+
+      def make_expando_fields(key)
+        key = key.to_s
+        fields = Array.new 
+        parts = key.split("_")
+        length = parts.length
+        parts.each_index do |i|
+          beginning = nil
+          remainder = nil
+          if i == 0
+            beginning = "X"
+          else
+            beginning = parts[0..i-1].join("_")
+          end
+
+          if i == length-1
+            remainder = "X"
+          else
+            remainder = parts[i+1..-1].join("_")
+          end
+
+          if beginning == "X" || remainder == "X"
+            unless beginning == "X" && remainder == "X"
+              fields << "#{beginning}_#{remainder}" 
+            end
+          else
+            fields << "#{beginning}_X_#{remainder}"
+          end
+        end
+        fields
+      end
+
+      def set_field_value(fields, key, value)
+        key = key.to_s
+        if fields.has_key?(key)
+          convert_field_to_array(fields, key, value) unless fields[key].kind_of?(Array)
+          add_value_to_field_array(fields, key, value)
+        else
+          check_value(value)
+          if value.kind_of?(Array)
+            fields[key] = Array.new
+            value.each do |v|
+              fields[key] << v.to_s
+            end
+          else
+            fields[key] = value.to_s
+          end
+        end
+        fields
+      end
+
+      def add_value_to_field_array(fields, key, value)
+        check_value(value)
+        if value.kind_of?(Array)
+          value.each do |v|
+            check_value(v)
+            fields[key] << v.to_s unless fields[key].include?(v.to_s)
+          end
+        else
+          fields[key] << value.to_s unless fields[key].include?(value.to_s)
+        end
+        fields
+      end
+
+      def convert_field_to_array(fields, key, value)
+        if fields[key] != value
+          safe = fields[key]
+          fields[key] = [ safe ]
+        end
+        fields
+      end
+
+      def check_value(value)
+        raise ArgumentError, "Value must not be a type of hash!" if value.kind_of?(Hash)
+        value
+      end
+
+    end
+  end
+end
+