chef-solr 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,78 @@
1
+ #
2
+ # Author:: Adam Jacob (<adam@opscode.com>)
3
+ # Copyright:: Copyright (c) 2009 Opscode, Inc.
4
+ # License:: Apache License, Version 2.0
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ require 'rubygems'
20
+ require 'chef/log'
21
+ require 'chef/config'
22
+ require 'chef/solr'
23
+ require 'chef/solr/index'
24
+ require 'chef/node'
25
+ require 'chef/role'
26
+ require 'chef/rest'
27
+ require 'chef/data_bag'
28
+ require 'chef/data_bag_item'
29
+ require 'chef/api_client'
30
+ require 'chef/couchdb'
31
+ require 'chef/index_queue'
32
+
33
+ class Chef
34
+ class Solr
35
+ class IndexQueueConsumer
36
+ include Chef::IndexQueue::Consumer
37
+
38
+ expose :add, :delete
39
+
40
+ def add(payload)
41
+ index = Chef::Solr::Index.new
42
+ Chef::Log.debug("Dequeued item for indexing: #{payload.inspect}")
43
+
44
+ response = begin
45
+ pitem = payload["item"].to_hash
46
+ generate_response { index.add(payload["id"], payload["database"], payload["type"], pitem) }
47
+ rescue NoMethodError
48
+ generate_response() { raise ArgumentError, "Payload item does not respond to :keys or :to_hash, cannot index!" }
49
+ end
50
+
51
+ Chef::Log.info("Indexing #{payload["type"]} #{payload["id"]} from #{payload["database"]} status #{response[:status]}#{response[:status] == :error ? ' ' + response[:error] : ''}")
52
+ response
53
+ end
54
+
55
+ def delete(payload)
56
+ response = generate_response { Chef::Solr::Index.new.delete(payload["id"]) }
57
+ Chef::Log.info("Removed #{payload["id"]} from the index")
58
+ response
59
+ end
60
+
61
+ private
62
+ def generate_response(&block)
63
+ response = {}
64
+ begin
65
+ block.call
66
+ rescue
67
+ response[:status] = :error
68
+ response[:error] = $!
69
+ else
70
+ response[:status] = :ok
71
+ end
72
+ response
73
+ end
74
+
75
+ end
76
+ end
77
+ end
78
+
@@ -0,0 +1,87 @@
1
+ #
2
+ # Author:: Adam Jacob (<adam@opscode.com>)
3
+ # Copyright:: Copyright (c) 2009 Opscode, Inc.
4
+ # License:: Apache License, Version 2.0
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ require 'chef/couchdb'
20
+ require 'chef/node'
21
+ require 'chef/role'
22
+ require 'chef/data_bag'
23
+ require 'chef/data_bag_item'
24
+ require 'chef/solr'
25
+ require 'chef/log'
26
+ require 'chef/config'
27
+
28
+ class Chef
29
+ class Solr
30
+ class Query < Chef::Solr
31
+
32
+ # Create a new Query object - takes the solr_url and optional
33
+ # couchdb_database to inflate objects into.
34
+ def initialize(solr_url=Chef::Config[:solr_url], database=Chef::Config[:couchdb_database])
35
+ super(solr_url)
36
+ @database = database
37
+ @couchdb = Chef::CouchDB.new(nil, database)
38
+ end
39
+
40
+ # A raw query against CouchDB - takes the type of object to find, and raw
41
+ # Solr options.
42
+ #
43
+ # You'll wind up having to page things yourself.
44
+ def raw(type, options={})
45
+ qtype = case type
46
+ when "role",:role,"node",:node,"client",:client
47
+ type
48
+ else
49
+ [ "data_bag_item", type ]
50
+ end
51
+ results = solr_select(@database, qtype, options)
52
+ Chef::Log.debug("Searching #{@database} #{qtype.inspect} for #{options.inspect} with results:\n#{results.inspect}")
53
+ objects = if results["response"]["docs"].length > 0
54
+ bulk_objects = @couchdb.bulk_get( results["response"]["docs"].collect { |d| d["X_CHEF_id_CHEF_X"] } )
55
+ Chef::Log.debug("bulk get of objects: #{bulk_objects.inspect}")
56
+ bulk_objects
57
+ else
58
+ []
59
+ end
60
+ [ objects, results["response"]["start"], results["response"]["numFound"], results["responseHeader"] ]
61
+ end
62
+
63
+ # Search Solr for objects of a given type, for a given query. If you give
64
+ # it a block, it will handle the paging for you dynamically.
65
+ def search(type, query="*:*", sort=nil, start=0, rows=20, &block)
66
+ options = {
67
+ :q => query,
68
+ :start => start,
69
+ :rows => rows
70
+ }
71
+ options[:sort] = sort if sort && ! sort.empty?
72
+ objects, start, total, response_header = raw(type, options)
73
+ if block
74
+ objects.each { |o| block.call(o) }
75
+ unless (start + objects.length) >= total
76
+ nstart = start + rows
77
+ search(type, query, sort, nstart, rows, &block)
78
+ end
79
+ true
80
+ else
81
+ [ objects, start, total ]
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end
87
+
Binary file
Binary file
@@ -0,0 +1,168 @@
1
+ require File.expand_path(File.join("#{File.dirname(__FILE__)}", '..', '..', 'spec_helper'))
2
+
3
+ describe Chef::Solr::Index do
4
+ before(:each) do
5
+ @index = Chef::Solr::Index.new
6
+ end
7
+
8
+ describe "initialize" do
9
+ it "should return a Chef::Solr::Index" do
10
+ @index.should be_a_kind_of(Chef::Solr::Index)
11
+ end
12
+ end
13
+
14
+ describe "add" do
15
+ before(:each) do
16
+ @index.stub!(:solr_add).and_return(true)
17
+ @index.stub!(:solr_commit).and_return(true)
18
+ end
19
+
20
+ it "should take an object that responds to .keys as it's argument" do
21
+ lambda { @index.add(1, "chef_opscode", "node", { :one => :two }) }.should_not raise_error(ArgumentError)
22
+ lambda { @index.add(1, "chef_opscode", "node", "SOUP") }.should raise_error(ArgumentError)
23
+ lambda { @index.add(2, "chef_opscode", "node", mock("Foo", :keys => true)) }.should_not raise_error(ArgumentError)
24
+ end
25
+
26
+ it "should index the object as a single flat hash, with only strings or arrays as values" do
27
+ validate = {
28
+ "X_CHEF_id_CHEF_X" => 1,
29
+ "X_CHEF_database_CHEF_X" => "monkey",
30
+ "X_CHEF_type_CHEF_X" => "snakes",
31
+ "foo" => "bar",
32
+ "battles" => [ "often", "but", "for" ],
33
+ "battles_often" => "sings like smurfs",
34
+ "often" => "sings like smurfs",
35
+ "battles_but" => "still has good records",
36
+ "but" => "still has good records",
37
+ "battles_for" => [ "all", "of", "that" ],
38
+ "for" => [ "all", "of", "that" ],
39
+ "snoopy" => "sits_in_a_barn",
40
+ "battles_X" => [ "sings like smurfs", "still has good records", "all", "of", "that" ],
41
+ "X_often" => "sings like smurfs",
42
+ "X_but" => "still has good records",
43
+ "X_for" => [ "all", "of", "that" ]
44
+ }
45
+ to_index = @index.add(1, "monkey", "snakes", {
46
+ "foo" => :bar,
47
+ "battles" => {
48
+ "often" => "sings like smurfs",
49
+ "but" => "still has good records",
50
+ "for" => [ "all", "of", "that" ]
51
+ },
52
+ "snoopy" => "sits_in_a_barn"
53
+ })
54
+ validate.each do |k, v|
55
+ if v.kind_of?(Array)
56
+ # Every entry in to_index[k] should be in v
57
+ r = to_index[k] & v
58
+ r.length.should == to_index[k].length
59
+ else
60
+ to_index[k].should == v
61
+ end
62
+ end
63
+ end
64
+
65
+ it "should send the document to solr" do
66
+ @index.should_receive(:solr_add)
67
+ @index.add(1, "monkey", "snakes", { "foo" => "bar" })
68
+ end
69
+ end
70
+
71
+ describe "delete" do
72
+ it "should delete by id" do
73
+ @index.should_receive(:solr_delete_by_id).with(1)
74
+ @index.delete(1)
75
+ end
76
+ end
77
+
78
+ describe "delete_by_query" do
79
+ it "should delete by query" do
80
+ @index.should_receive(:solr_delete_by_query).with("foo:bar")
81
+ @index.delete_by_query("foo:bar")
82
+ end
83
+ end
84
+
85
+ describe "flatten_and_expand" do
86
+ before(:each) do
87
+ @fields = Hash.new
88
+ end
89
+
90
+ it "should set a value for the parent as key, with the key as the value" do
91
+ @index.flatten_and_expand({ "one" => "woot" }, @fields, "omerta")
92
+ @fields["omerta"].should == "one"
93
+ end
94
+
95
+ it "should call itself recursively for values that are hashes" do
96
+ @index.flatten_and_expand({ "one" => { "two" => "three", "four" => { "five" => "six" } }}, @fields)
97
+ {
98
+ "one" => [ "two", "four" ],
99
+ "one_two" => "three",
100
+ "X_two" => "three",
101
+ "two" => "three",
102
+ "one_four" => "five",
103
+ "X_four" => "five",
104
+ "one_X" => [ "three", "five" ],
105
+ "one_four_five" => "six",
106
+ "X_four_five" => "six",
107
+ "one_X_five" => "six",
108
+ "one_four_X" => "six",
109
+ "five" => "six"
110
+ }.each do |k, v|
111
+ @fields[k].should == v
112
+ end
113
+ end
114
+
115
+ end
116
+
117
+ describe "set_field_value" do
118
+ before(:each) do
119
+ @fields = Hash.new
120
+ end
121
+
122
+ it "should set a value in the fields hash" do
123
+ @index.set_field_value(@fields, "one", "two")
124
+ @fields["one"].should eql("two")
125
+ end
126
+
127
+ it "should create an array of all values, if a field is set twice" do
128
+ @index.set_field_value(@fields, "one", "two")
129
+ @index.set_field_value(@fields, "one", "three")
130
+ @fields["one"].should eql([ "two", "three" ])
131
+ end
132
+
133
+ it "should not add duplicate values to a field when there is one string entry" do
134
+ @index.set_field_value(@fields, "one", "two")
135
+ @index.set_field_value(@fields, "one", "two")
136
+ @fields["one"].should eql("two")
137
+ end
138
+
139
+ it "should not add duplicate values to a field when it is an array" do
140
+ @index.set_field_value(@fields, "one", "two")
141
+ @index.set_field_value(@fields, "one", "three")
142
+ @index.set_field_value(@fields, "one", "two")
143
+ @fields["one"].should eql([ "two", "three" ])
144
+ end
145
+
146
+ it "should accept arrays as values" do
147
+ @index.set_field_value(@fields, "one", [ "two", "three" ])
148
+ @fields["one"].should eql([ "two", "three" ])
149
+ end
150
+
151
+ it "should not duplicate values when a field has been set with multiple arrays" do
152
+ @index.set_field_value(@fields, "one", [ "two", "three" ])
153
+ @index.set_field_value(@fields, "one", [ "two", "four" ])
154
+ @fields["one"].should eql([ "two", "three", "four" ])
155
+ end
156
+
157
+
158
+ it "should allow you to set a value in the fields hash to an array" do
159
+ @index.set_field_value(@fields, "one", [ "foo", "bar", "baz" ])
160
+ end
161
+
162
+ it "should not allow you to set a value in the fields hash to a hash" do
163
+ lambda {
164
+ @index.set_field_value(@fields, "one", { "two" => "three" })
165
+ }.should raise_error(ArgumentError)
166
+ end
167
+ end
168
+ end
@@ -0,0 +1,14 @@
1
+ require File.expand_path(File.join("#{File.dirname(__FILE__)}", '..', '..', 'spec_helper'))
2
+
3
+ describe Chef::Solr::Query do
4
+ before(:each) do
5
+ @query = Chef::Solr::Query.new
6
+ end
7
+
8
+ describe "initialize" do
9
+ it "should return a Chef::Solr::Query" do
10
+ @query.should be_a_kind_of(Chef::Solr::Query)
11
+ end
12
+ end
13
+ end
14
+
@@ -0,0 +1,174 @@
1
+ require File.expand_path(File.join("#{File.dirname(__FILE__)}", '..', 'spec_helper'))
2
+
3
+ describe Chef::Solr do
4
+ before(:each) do
5
+ @solr = Chef::Solr.new
6
+ end
7
+
8
+ describe "initialize" do
9
+ it "should create a new Chef::Solr object" do
10
+ @solr.should be_a_kind_of(Chef::Solr)
11
+ end
12
+
13
+ it "should accept an alternate solr url" do
14
+ solr = Chef::Solr.new("http://example.com")
15
+ solr.solr_url.should eql("http://example.com")
16
+ end
17
+ end
18
+
19
+ describe "solr_select" do
20
+ before(:each) do
21
+ @http_response = mock(
22
+ "Net::HTTP::Response",
23
+ :kind_of? => Net::HTTPSuccess,
24
+ :body => "{ :some => :hash }"
25
+ )
26
+ @http = mock("Net::HTTP", :request => @http_response)
27
+ @solr.http = @http
28
+ end
29
+
30
+ it "should call get to /solr/select with the escaped query" do
31
+ Net::HTTP::Get.should_receive(:new).with(%r(q=hostname%3Alatte))
32
+ @solr.solr_select("chef_opscode", "node", :q => "hostname:latte")
33
+ end
34
+
35
+ it "should call get to /solr/select with wt=ruby" do
36
+ Net::HTTP::Get.should_receive(:new).with(%r(wt=ruby))
37
+ @solr.solr_select("chef_opscode", "node", :q => "hostname:latte")
38
+ end
39
+
40
+ it "should call get to /solr/select with indent=off" do
41
+ Net::HTTP::Get.should_receive(:new).with(%r(indent=off))
42
+ @solr.solr_select("chef_opscode", "node", :q => "hostname:latte")
43
+ end
44
+
45
+ it "should call get to /solr/select with filter query" do
46
+ Net::HTTP::Get.should_receive(:new).with(/fq=%2BX_CHEF_database_CHEF_X%3Achef_opscode\+%2BX_CHEF_type_CHEF_X%3Anode/)
47
+ @solr.solr_select("chef_opscode", "node", :q => "hostname:latte")
48
+ end
49
+
50
+ it "should return the evaluated response body" do
51
+ res = @solr.solr_select("chef_opscode", "node", :q => "hostname:latte")
52
+ res.should == { :some => :hash }
53
+ end
54
+ end
55
+
56
+
57
+ describe "post_to_solr" do
58
+ before(:each) do
59
+ @http_response = mock(
60
+ "Net::HTTP::Response",
61
+ :kind_of? => Net::HTTPSuccess,
62
+ :body => "{ :some => :hash }"
63
+ )
64
+ @http_request = mock(
65
+ "Net::HTTP::Request",
66
+ :body= => true
67
+ )
68
+ @http = mock("Net::HTTP", :request => @http_response)
69
+ @solr.http = @http
70
+ Net::HTTP::Post.stub!(:new).and_return(@http_request)
71
+ @doc = { "foo" => "bar" }
72
+ end
73
+
74
+ it "should post to /solr/update" do
75
+ Net::HTTP::Post.should_receive(:new).with("/solr/update", "Content-Type" => "text/xml").and_return(@http_request)
76
+ @solr.post_to_solr(@doc)
77
+ end
78
+
79
+ it "should set the body of the request to the stringified doc" do
80
+ @http_request.should_receive(:body=).with("foo")
81
+ @solr.post_to_solr(:foo)
82
+ end
83
+
84
+ it "should send the request to solr" do
85
+ @http.should_receive(:request).with(@http_request).and_return(@http_response)
86
+ @solr.post_to_solr(:foo)
87
+ end
88
+ end
89
+
90
+ describe "solr_add" do
91
+ before(:each) do
92
+ @solr.stub!(:post_to_solr).and_return(true)
93
+ @data = { "foo" => "bar" }
94
+ end
95
+
96
+ it "should send valid XML to solr" do
97
+ @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<add><doc><field name=\"foo\">bar</field></doc></add>\n")
98
+ @solr.solr_add(@data)
99
+ end
100
+
101
+ it "XML escapes content before sending to SOLR" do
102
+ @data["foo"] = "<&>"
103
+ @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<add><doc><field name=\"foo\">&lt;&amp;&gt;</field></doc></add>\n")
104
+
105
+ @solr.solr_add(@data)
106
+ end
107
+ end
108
+
109
+ describe "solr_commit" do
110
+ before(:each) do
111
+ @solr.stub!(:post_to_solr).and_return(true)
112
+ end
113
+
114
+ it "should send valid commit xml to solr" do
115
+ @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<commit/>\n")
116
+ @solr.solr_commit
117
+ end
118
+ end
119
+
120
+ describe "solr_optimize" do
121
+ before(:each) do
122
+ @solr.stub!(:post_to_solr).and_return(true)
123
+ end
124
+
125
+ it "should send valid commit xml to solr" do
126
+ @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<optimize/>\n")
127
+ @solr.solr_optimize
128
+ end
129
+ end
130
+
131
+ describe "solr_rollback" do
132
+ before(:each) do
133
+ @solr.stub!(:post_to_solr).and_return(true)
134
+ end
135
+
136
+ it "should send valid commit xml to solr" do
137
+ @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<rollback/>\n")
138
+ @solr.solr_rollback
139
+ end
140
+ end
141
+
142
+ describe "solr_delete_by_id" do
143
+ before(:each) do
144
+ @solr.stub!(:post_to_solr).and_return(true)
145
+ end
146
+
147
+ it "should send valid delete id xml to solr" do
148
+ @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><id>1</id></delete>\n")
149
+ @solr.solr_delete_by_id(1)
150
+ end
151
+
152
+ it "should accept multiple ids" do
153
+ @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><id>1</id><id>2</id></delete>\n")
154
+ @solr.solr_delete_by_id([ 1, 2 ])
155
+ end
156
+ end
157
+
158
+ describe "solr_delete_by_query" do
159
+ before(:each) do
160
+ @solr.stub!(:post_to_solr).and_return(true)
161
+ end
162
+
163
+ it "should send valid delete id xml to solr" do
164
+ @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><query>foo:bar</query></delete>\n")
165
+ @solr.solr_delete_by_query("foo:bar")
166
+ end
167
+
168
+ it "should accept multiple ids" do
169
+ @solr.should_receive(:post_to_solr).with("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<delete><query>foo:bar</query><query>baz:bum</query></delete>\n")
170
+ @solr.solr_delete_by_query([ "foo:bar", "baz:bum" ])
171
+ end
172
+ end
173
+
174
+ end