chef-solr 0.8.2

data/lib/chef/solr/index_queue_consumer.rb

@@ -0,0 +1,78 @@
+#
+# Author:: Adam Jacob (<adam@opscode.com>)
+# Copyright:: Copyright (c) 2009 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'rubygems'
+require 'chef/log'
+require 'chef/config'
+require 'chef/solr'
+require 'chef/solr/index'
+require 'chef/node'
+require 'chef/role'
+require 'chef/rest'
+require 'chef/data_bag'
+require 'chef/data_bag_item'
+require 'chef/api_client'
+require 'chef/couchdb'
+require 'chef/index_queue'
+
+class Chef
+  class Solr
+    class IndexQueueConsumer
+      include Chef::IndexQueue::Consumer
+
+      expose :add, :delete
+      
+      def add(payload)
+        index = Chef::Solr::Index.new
+        Chef::Log.debug("Dequeued item for indexing: #{payload.inspect}")
+
+        response = begin
+                     pitem = payload["item"].to_hash                  
+                     generate_response { index.add(payload["id"], payload["database"], payload["type"], pitem) }                  
+                   rescue NoMethodError
+                     generate_response() { raise ArgumentError, "Payload item does not respond to :keys or :to_hash, cannot index!" }
+                   end
+        
+        Chef::Log.info("Indexing #{payload["type"]} #{payload["id"]} from #{payload["database"]} status #{response[:status]}#{response[:status] == :error ? ' ' + response[:error] : ''}")
+        response 
+      end
+
+      def delete(payload)
+        response = generate_response { Chef::Solr::Index.new.delete(payload["id"]) }
+        Chef::Log.info("Removed #{payload["id"]} from the index")
+        response
+      end
+      
+      private
+        def generate_response(&block)
+          response = {}
+          begin
+            block.call
+          rescue
+            response[:status] = :error
+            response[:error] = $!
+          else
+            response[:status] = :ok
+          end
+          response
+        end
+
+    end
+  end
+end
+

data/lib/chef/solr/query.rb

@@ -0,0 +1,87 @@
+#
+# Author:: Adam Jacob (<adam@opscode.com>)
+# Copyright:: Copyright (c) 2009 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/couchdb'
+require 'chef/node'
+require 'chef/role'
+require 'chef/data_bag'
+require 'chef/data_bag_item'
+require 'chef/solr'
+require 'chef/log'
+require 'chef/config'
+
+class Chef
+  class Solr
+    class Query < Chef::Solr
+      
+      # Create a new Query object - takes the solr_url and optional
+      # couchdb_database to inflate objects into.
+      def initialize(solr_url=Chef::Config[:solr_url], database=Chef::Config[:couchdb_database])
+        super(solr_url)
+        @database = database
+        @couchdb = Chef::CouchDB.new(nil, database)
+      end
+
+      # A raw query against CouchDB - takes the type of object to find, and raw
+      # Solr options.
+      #
+      # You'll wind up having to page things yourself.
+      def raw(type, options={})
+        qtype = case type
+                when "role",:role,"node",:node,"client",:client
+                  type
+                else
+                  [ "data_bag_item", type ]
+                end
+        results = solr_select(@database, qtype, options)
+        Chef::Log.debug("Searching #{@database} #{qtype.inspect} for #{options.inspect} with results:\n#{results.inspect}") 
+        objects = if results["response"]["docs"].length > 0
+                    bulk_objects = @couchdb.bulk_get( results["response"]["docs"].collect { |d| d["X_CHEF_id_CHEF_X"] } )
+                    Chef::Log.debug("bulk get of objects: #{bulk_objects.inspect}")
+                    bulk_objects
+                  else
+                    []
+                  end
+        [ objects, results["response"]["start"], results["response"]["numFound"], results["responseHeader"] ] 
+      end
+
+      # Search Solr for objects of a given type, for a given query. If you give
+      # it a block, it will handle the paging for you dynamically.
+      def search(type, query="*:*", sort=nil, start=0, rows=20, &block)
+        options = {
+          :q => query,
+          :start => start,
+          :rows => rows 
+        }
+        options[:sort] = sort if sort && ! sort.empty?
+        objects, start, total, response_header = raw(type, options)
+        if block
+          objects.each { |o| block.call(o) }
+          unless (start + objects.length) >= total
+            nstart = start + rows
+            search(type, query, sort, nstart, rows, &block)
+          end
+          true
+        else
+          [ objects, start, total ]
+        end
+      end
+    end
+  end
+end
+

data/spec/chef/solr/index_spec.rb

@@ -0,0 +1,168 @@
+require File.expand_path(File.join("#{File.dirname(__FILE__)}", '..', '..', 'spec_helper'))
+
+describe Chef::Solr::Index do
+  before(:each) do
+    @index = Chef::Solr::Index.new
+  end
+
+  describe "initialize" do
+    it "should return a Chef::Solr::Index" do
+      @index.should be_a_kind_of(Chef::Solr::Index)
+    end
+  end
+
+  describe "add" do
+    before(:each) do
+      @index.stub!(:solr_add).and_return(true)
+      @index.stub!(:solr_commit).and_return(true)
+    end
+
+    it "should take an object that responds to .keys as it's argument" do
+      lambda { @index.add(1, "chef_opscode", "node", { :one => :two }) }.should_not raise_error(ArgumentError)
+      lambda { @index.add(1, "chef_opscode", "node", "SOUP") }.should raise_error(ArgumentError)
+      lambda { @index.add(2, "chef_opscode", "node", mock("Foo", :keys => true)) }.should_not raise_error(ArgumentError)
+    end
+
+    it "should index the object as a single flat hash, with only strings or arrays as values" do
+      validate = {
+        "X_CHEF_id_CHEF_X" => 1,
+        "X_CHEF_database_CHEF_X" => "monkey",
+        "X_CHEF_type_CHEF_X" => "snakes", 
+        "foo" => "bar",
+        "battles" => [ "often", "but", "for" ],
+        "battles_often" => "sings like smurfs",
+        "often" => "sings like smurfs",
+        "battles_but" => "still has good records",
+        "but" => "still has good records",
+        "battles_for" => [ "all", "of", "that" ],
+        "for" => [ "all", "of", "that" ],
+        "snoopy" => "sits_in_a_barn",
+        "battles_X" => [ "sings like smurfs", "still has good records", "all", "of", "that" ],
+        "X_often" => "sings like smurfs",
+        "X_but" => "still has good records",
+        "X_for" => [ "all", "of", "that" ]
+      } 
+      to_index = @index.add(1, "monkey", "snakes", { 
+        "foo" => :bar,
+        "battles" => { 
+          "often" => "sings like smurfs",
+          "but" => "still has good records",
+          "for" => [ "all", "of", "that" ]
+        },
+        "snoopy" => "sits_in_a_barn"
+      })
+      validate.each do |k, v|
+        if v.kind_of?(Array)
+          # Every entry in to_index[k] should be in v
+          r = to_index[k] & v
+          r.length.should == to_index[k].length
+        else
+          to_index[k].should == v
+        end
+      end
+    end
+
+    it "should send the document to solr" do
+      @index.should_receive(:solr_add)
+      @index.add(1, "monkey", "snakes", { "foo" => "bar" })
+    end
+  end
+
+  describe "delete" do
+    it "should delete by id" do
+      @index.should_receive(:solr_delete_by_id).with(1)
+      @index.delete(1)
+    end
+  end
+
+  describe "delete_by_query" do
+    it "should delete by query" do
+      @index.should_receive(:solr_delete_by_query).with("foo:bar")
+      @index.delete_by_query("foo:bar")
+    end
+  end
+
+  describe "flatten_and_expand" do
+    before(:each) do
+      @fields = Hash.new
+    end
+
+    it "should set a value for the parent as key, with the key as the value" do
+      @index.flatten_and_expand({ "one" => "woot" }, @fields, "omerta")
+      @fields["omerta"].should == "one"
+    end
+
+    it "should call itself recursively for values that are hashes" do
+      @index.flatten_and_expand({ "one" => { "two" => "three", "four" => { "five" => "six" } }}, @fields)
+      {
+        "one" => [ "two", "four" ],
+        "one_two" => "three",
+        "X_two" => "three",
+        "two" => "three",
+        "one_four" => "five",
+        "X_four" => "five",
+        "one_X" => [ "three", "five" ],
+        "one_four_five" => "six", 
+        "X_four_five" => "six",
+        "one_X_five" => "six",
+        "one_four_X" => "six",
+        "five" => "six"
+      }.each do |k, v|
+        @fields[k].should == v
+      end
+    end
+
+  end
+
+  describe "set_field_value" do
+    before(:each) do
+      @fields = Hash.new
+    end
+
+    it "should set a value in the fields hash" do
+      @index.set_field_value(@fields, "one", "two")
+      @fields["one"].should eql("two")
+    end
+
+    it "should create an array of all values, if a field is set twice" do
+      @index.set_field_value(@fields, "one", "two")
+      @index.set_field_value(@fields, "one", "three")
+      @fields["one"].should eql([ "two", "three" ])
+    end
+
+    it "should not add duplicate values to a field when there is one string entry" do
+      @index.set_field_value(@fields, "one", "two")
+      @index.set_field_value(@fields, "one", "two")
+      @fields["one"].should eql("two")
+    end
+
+    it "should not add duplicate values to a field when it is an array" do
+      @index.set_field_value(@fields, "one", "two")
+      @index.set_field_value(@fields, "one", "three")
+      @index.set_field_value(@fields, "one", "two")
+      @fields["one"].should eql([ "two", "three" ])
+    end
+
+    it "should accept arrays as values" do
+      @index.set_field_value(@fields, "one", [ "two", "three" ])
+      @fields["one"].should eql([ "two", "three" ]) 
+    end
+
+    it "should not duplicate values when a field has been set with multiple arrays" do
+      @index.set_field_value(@fields, "one", [ "two", "three" ])
+      @index.set_field_value(@fields, "one", [ "two", "four" ])
+      @fields["one"].should eql([ "two", "three", "four" ]) 
+    end
+
+
+    it "should allow you to set a value in the fields hash to an array" do
+      @index.set_field_value(@fields, "one", [ "foo", "bar", "baz" ])
+    end
+
+    it "should not allow you to set a value in the fields hash to a hash" do
+      lambda {
+        @index.set_field_value(@fields, "one", { "two" => "three" })
+      }.should raise_error(ArgumentError)
+    end
+  end
+end

data/spec/chef/solr/query_spec.rb

@@ -0,0 +1,14 @@
+require File.expand_path(File.join("#{File.dirname(__FILE__)}", '..', '..', 'spec_helper'))
+
+describe Chef::Solr::Query do
+  before(:each) do
+    @query = Chef::Solr::Query.new
+  end
+
+  describe "initialize" do
+    it "should return a Chef::Solr::Query" do
+      @query.should be_a_kind_of(Chef::Solr::Query)
+    end
+  end
+end
+

data/spec/chef/solr_spec.rb

@@ -0,0 +1,174 @@
+require File.expand_path(File.join("#{File.dirname(__FILE__)}", '..', 'spec_helper'))
+
+describe Chef::Solr do
+  before(:each) do 
+    @solr = Chef::Solr.new
+  end
+
+  describe "initialize" do
+    it "should create a new Chef::Solr object" do
+      @solr.should be_a_kind_of(Chef::Solr)
+    end
+
+    it "should accept an alternate solr url" do
+      solr = Chef::Solr.new("http://example.com")
+      solr.solr_url.should eql("http://example.com")
+    end
+  end
+
+  describe "solr_select" do
+    before(:each) do
+      @http_response = mock(
+        "Net::HTTP::Response", 
+        :kind_of? => Net::HTTPSuccess,
+        :body => "{ :some => :hash }"
+      )
+      @http = mock("Net::HTTP", :request => @http_response)
+      @solr.http = @http
+    end
+
+    it "should call get to /solr/select with the escaped query" do
+      Net::HTTP::Get.should_receive(:new).with(%r(q=hostname%3Alatte))
+      @solr.solr_select("chef_opscode", "node", :q => "hostname:latte")
+    end
+
+    it "should call get to /solr/select with wt=ruby" do
+      Net::HTTP::Get.should_receive(:new).with(%r(wt=ruby))
+      @solr.solr_select("chef_opscode", "node", :q => "hostname:latte")
+    end
+
+    it "should call get to /solr/select with indent=off" do
+      Net::HTTP::Get.should_receive(:new).with(%r(indent=off))
+      @solr.solr_select("chef_opscode", "node", :q => "hostname:latte")
+    end
+     
+    it "should call get to /solr/select with filter query" do
+      Net::HTTP::Get.should_receive(:new).with(/fq=%2BX_CHEF_database_CHEF_X%3Achef_opscode\+%2BX_CHEF_type_CHEF_X%3Anode/)
+      @solr.solr_select("chef_opscode", "node", :q => "hostname:latte")
+    end
+
+    it "should return the evaluated response body" do
+      res = @solr.solr_select("chef_opscode", "node", :q => "hostname:latte")
+      res.should == { :some => :hash }
+    end
+  end
+
+
+  describe "post_to_solr" do
+    before(:each) do
+      @http_response = mock(
+        "Net::HTTP::Response", 
+        :kind_of? => Net::HTTPSuccess,
+        :body => "{ :some => :hash }"
+      )
+      @http_request = mock(
+        "Net::HTTP::Request",
+        :body= => true
+      )
+      @http = mock("Net::HTTP", :request => @http_response)
+      @solr.http = @http
+      Net::HTTP::Post.stub!(:new).and_return(@http_request)
+      @doc = { "foo" => "bar" }
+    end
+  
+    it "should post to /solr/update" do
+      Net::HTTP::Post.should_receive(:new).with("/solr/update", "Content-Type" => "text/xml").and_return(@http_request)
+      @solr.post_to_solr(@doc)
+    end
+
+    it "should set the body of the request to the stringified doc" do
+      @http_request.should_receive(:body=).with("foo")
+      @solr.post_to_solr(:foo)
+    end
+
+    it "should send the request to solr" do
+      @http.should_receive(:request).with(@http_request).and_return(@http_response)
+      @solr.post_to_solr(:foo)
+    end
+  end
+
+  describe "solr_add" do
+    before(:each) do
+      @solr.stub!(:post_to_solr).and_return(true)
+      @data = { "foo" => "bar" }
+    end
+
+    it "should send valid XML to solr" do
+      @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<add><doc><field name=\"foo\">bar</field></doc></add>\n")
+      @solr.solr_add(@data)
+    end
+
+    it "XML escapes content before sending to SOLR" do
+      @data["foo"] = "<&>"
+      @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<add><doc><field name=\"foo\">&lt;&amp;&gt;</field></doc></add>\n")
+
+      @solr.solr_add(@data)
+    end
+  end
+
+  describe "solr_commit" do
+    before(:each) do
+      @solr.stub!(:post_to_solr).and_return(true)
+    end
+
+    it "should send valid commit xml to solr" do
+      @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<commit/>\n")
+      @solr.solr_commit
+    end
+  end
+
+  describe "solr_optimize" do
+    before(:each) do
+      @solr.stub!(:post_to_solr).and_return(true)
+    end
+
+    it "should send valid commit xml to solr" do
+      @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<optimize/>\n")
+      @solr.solr_optimize
+    end
+  end
+
+  describe "solr_rollback" do
+    before(:each) do
+      @solr.stub!(:post_to_solr).and_return(true)
+    end
+
+    it "should send valid commit xml to solr" do
+      @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<rollback/>\n")
+      @solr.solr_rollback
+    end
+  end
+
+  describe "solr_delete_by_id" do
+    before(:each) do
+      @solr.stub!(:post_to_solr).and_return(true)
+    end
+
+    it "should send valid delete id xml to solr" do
+      @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><id>1</id></delete>\n")
+      @solr.solr_delete_by_id(1)
+    end
+
+    it "should accept multiple ids" do
+      @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><id>1</id><id>2</id></delete>\n")
+      @solr.solr_delete_by_id([ 1, 2 ])
+    end
+  end
+
+  describe "solr_delete_by_query" do
+    before(:each) do
+      @solr.stub!(:post_to_solr).and_return(true)
+    end
+
+    it "should send valid delete id xml to solr" do
+      @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><query>foo:bar</query></delete>\n")
+      @solr.solr_delete_by_query("foo:bar")
+    end
+
+    it "should accept multiple ids" do
+      @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><query>foo:bar</query><query>baz:bum</query></delete>\n")
+      @solr.solr_delete_by_query([ "foo:bar", "baz:bum" ])
+    end
+  end
+
+end