mdwan-rsolr 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGES.txt +157 -0
- data/LICENSE +67 -0
- data/README.rdoc +126 -0
- data/Rakefile +39 -0
- data/examples/direct.rb +27 -0
- data/examples/http.rb +22 -0
- data/lib/core_ext.rb +25 -0
- data/lib/mash.rb +148 -0
- data/lib/rsolr.rb +52 -0
- data/lib/rsolr/adapter.rb +6 -0
- data/lib/rsolr/adapter/direct.rb +86 -0
- data/lib/rsolr/adapter/http.rb +44 -0
- data/lib/rsolr/connection.rb +107 -0
- data/lib/rsolr/http_client.rb +140 -0
- data/lib/rsolr/http_client/adapter.rb +6 -0
- data/lib/rsolr/http_client/adapter/curb.rb +51 -0
- data/lib/rsolr/http_client/adapter/net_http.rb +48 -0
- data/lib/rsolr/message.rb +153 -0
- data/test/connection/direct_test.rb +25 -0
- data/test/connection/http_test.rb +18 -0
- data/test/connection/test_methods.rb +97 -0
- data/test/http_client/curb_test.rb +18 -0
- data/test/http_client/net_http_test.rb +12 -0
- data/test/http_client/test_methods.rb +40 -0
- data/test/http_client/util_test.rb +40 -0
- data/test/message_test.rb +107 -0
- metadata +93 -0
@@ -0,0 +1,140 @@
|
|
1
|
+
# A simple wrapper for different http client implementations.
|
2
|
+
# Supports #get and #post
|
3
|
+
# This was motivated by: http://apocryph.org/2008/11/09/more_indepth_analysis_ruby_http_client_performance/
|
4
|
+
# Curb is the default adapter
|
5
|
+
|
6
|
+
# Each adapters response should be a hash with the following keys:
|
7
|
+
# :status_code
|
8
|
+
# :url
|
9
|
+
# :body
|
10
|
+
# :path
|
11
|
+
# :params
|
12
|
+
# :data
|
13
|
+
# :headers
|
14
|
+
|
15
|
+
# Example:
|
16
|
+
# connector = RSolr::HTTPClient.Connector.new
|
17
|
+
# connector.adapter_name = :net_http # switch to Net::HTTP before calling "connect"
|
18
|
+
# hclient = connector.connect('http://www.google.com')
|
19
|
+
# response = hclient.get('/search', :hl=>:en, :q=>:ruby, :btnG=>:Search)
|
20
|
+
# puts response[:status_code]
|
21
|
+
# puts response[:body]
|
22
|
+
|
23
|
+
require 'uri'
|
24
|
+
|
25
|
+
module RSolr::HTTPClient
|
26
|
+
|
27
|
+
autoload :Adapter, 'rsolr/http_client/adapter'
|
28
|
+
|
29
|
+
class UnkownAdapterError < RuntimeError; end
|
30
|
+
|
31
|
+
class Connector
|
32
|
+
|
33
|
+
attr_accessor :adapter_name
|
34
|
+
|
35
|
+
def initialize(adapter_name = :curb)
|
36
|
+
@adapter_name = adapter_name
|
37
|
+
end
|
38
|
+
|
39
|
+
def connect(url)
|
40
|
+
case adapter_name
|
41
|
+
when :curb
|
42
|
+
klass = 'Curb'
|
43
|
+
when :net_http
|
44
|
+
klass = 'NetHTTP'
|
45
|
+
else
|
46
|
+
raise UnkownAdapterError.new("Name: #{adapter_name}")
|
47
|
+
end
|
48
|
+
begin
|
49
|
+
RSolr::HTTPClient::Base.new RSolr::HTTPClient::Adapter.const_get(klass).new(url)
|
50
|
+
rescue ::URI::InvalidURIError
|
51
|
+
raise "#{$!} == #{url}"
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
class Base
|
58
|
+
|
59
|
+
attr_reader :adapter
|
60
|
+
|
61
|
+
def initialize(adapter)
|
62
|
+
@adapter = adapter
|
63
|
+
end
|
64
|
+
|
65
|
+
def get(path, params={})
|
66
|
+
begin
|
67
|
+
http_context = @adapter.get(path, params)
|
68
|
+
rescue
|
69
|
+
raise RSolr::RequestError.new($!)
|
70
|
+
end
|
71
|
+
http_context
|
72
|
+
end
|
73
|
+
|
74
|
+
def post(path, data, params={}, headers={})
|
75
|
+
begin
|
76
|
+
http_context = @adapter.post(path, data, params, headers)
|
77
|
+
rescue
|
78
|
+
raise RSolr::RequestError.new($!)
|
79
|
+
end
|
80
|
+
http_context
|
81
|
+
end
|
82
|
+
|
83
|
+
end
|
84
|
+
|
85
|
+
module Util
|
86
|
+
|
87
|
+
# escapes a query key/value for http
|
88
|
+
def escape(s)
|
89
|
+
s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
|
90
|
+
'%'+$1.unpack('H2'*$1.size).join('%').upcase
|
91
|
+
}.tr(' ', '+')
|
92
|
+
end
|
93
|
+
|
94
|
+
def build_url(url='', params={}, string_query='')
|
95
|
+
queries = [string_query, hash_to_params(params)]
|
96
|
+
queries.delete_if{|i| i.to_s.empty?}
|
97
|
+
url += "?#{queries.join('&')}" unless queries.empty?
|
98
|
+
url
|
99
|
+
end
|
100
|
+
|
101
|
+
def build_param(k,v)
|
102
|
+
"#{escape(k)}=#{escape(v)}"
|
103
|
+
end
|
104
|
+
|
105
|
+
#
|
106
|
+
# converts hash into URL query string, keys get an alpha sort
|
107
|
+
# if a value is an array, the array values get mapped to the same key:
|
108
|
+
# hash_to_params(:q=>'blah', :fq=>['blah', 'blah'], :facet=>{:field=>['location_facet', 'format_facet']})
|
109
|
+
# returns:
|
110
|
+
# ?q=blah&fq=blah&fq=blah&facet.field=location_facet&facet.field=format.facet
|
111
|
+
#
|
112
|
+
# if a value is empty/nil etc., the key is not added
|
113
|
+
def hash_to_params(params)
|
114
|
+
return unless params.is_a?(Hash)
|
115
|
+
# copy params and convert keys to strings
|
116
|
+
params = params.inject({}){|acc,(k,v)| acc.merge({k.to_s, v}) }
|
117
|
+
# get sorted keys
|
118
|
+
params.keys.sort.inject([]) do |acc,k|
|
119
|
+
v = params[k]
|
120
|
+
if v.is_a?(Array)
|
121
|
+
acc << v.reject{|i|i.to_s.empty?}.collect{|vv|build_param(k, vv)}
|
122
|
+
elsif v.is_a?(Hash)
|
123
|
+
# NOT USED
|
124
|
+
# creates dot based params like:
|
125
|
+
# hash_to_params(:facet=>{:field=>['one', 'two']}) == facet.field=one&facet.field=two
|
126
|
+
# TODO: should this go into a non-solr based param builder?
|
127
|
+
# - dotted syntax is special to solr only
|
128
|
+
#v.each_pair do |field,field_value|
|
129
|
+
# acc.push(hash_to_params({"#{k}.#{field}"=>field_value}))
|
130
|
+
#end
|
131
|
+
elsif ! v.to_s.empty?
|
132
|
+
acc.push(build_param(k, v))
|
133
|
+
end
|
134
|
+
acc
|
135
|
+
end.join('&')
|
136
|
+
end
|
137
|
+
|
138
|
+
end
|
139
|
+
|
140
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'curb'
|
3
|
+
|
4
|
+
class RSolr::HTTPClient::Adapter::Curb
|
5
|
+
|
6
|
+
include RSolr::HTTPClient::Util
|
7
|
+
|
8
|
+
attr :uri
|
9
|
+
attr :connection
|
10
|
+
|
11
|
+
def initialize(url)
|
12
|
+
@uri = URI.parse(url)
|
13
|
+
@connection = ::Curl::Easy.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def get(path, params={})
|
17
|
+
@connection.url = _build_url(path, params)
|
18
|
+
@connection.multipart_form_post = false
|
19
|
+
@connection.perform
|
20
|
+
create_http_context(path, params)
|
21
|
+
end
|
22
|
+
|
23
|
+
def post(path, data, params={}, headers={})
|
24
|
+
@connection.url = _build_url(path, params)
|
25
|
+
@connection.headers = headers
|
26
|
+
@connection.http_post(data)
|
27
|
+
create_http_context(path, params, data, headers)
|
28
|
+
end
|
29
|
+
|
30
|
+
protected
|
31
|
+
|
32
|
+
def create_http_context(path, params, data=nil, headers={})
|
33
|
+
{
|
34
|
+
:status_code=>@connection.response_code.to_i,
|
35
|
+
:url=>@connection.url,
|
36
|
+
:body=>@connection.body_str,
|
37
|
+
:path=>path,
|
38
|
+
:params=>params,
|
39
|
+
:data=>data,
|
40
|
+
:headers=>headers
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
def _build_url(path, params={})
|
45
|
+
url = @uri.scheme + '://' + @uri.host
|
46
|
+
url += ':' + @uri.port.to_s if @uri.port
|
47
|
+
url += @uri.path + path
|
48
|
+
build_url(url, params, @uri.query) # build_url is coming from RSolr::HTTPClient::Util
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
|
3
|
+
class RSolr::HTTPClient::Adapter::NetHTTP
|
4
|
+
|
5
|
+
include RSolr::HTTPClient::Util
|
6
|
+
|
7
|
+
attr :uri
|
8
|
+
attr :connection
|
9
|
+
|
10
|
+
def initialize(url)
|
11
|
+
@uri = URI.parse(url)
|
12
|
+
@connection = Net::HTTP.new(@uri.host, @uri.port)
|
13
|
+
end
|
14
|
+
|
15
|
+
def get(path, params={})
|
16
|
+
url = _build_url(path, params)
|
17
|
+
net_http_response = @connection.get(url)
|
18
|
+
create_http_context(net_http_response, url, path, params)
|
19
|
+
end
|
20
|
+
|
21
|
+
def post(path, data, params={}, headers={})
|
22
|
+
url = _build_url(path, params)
|
23
|
+
net_http_response = @connection.post(url, data, headers)
|
24
|
+
create_http_context(net_http_response, url, path, params, data, headers)
|
25
|
+
end
|
26
|
+
|
27
|
+
protected
|
28
|
+
|
29
|
+
def create_http_context(net_http_response, url, path, params, data=nil, headers={})
|
30
|
+
full_url = "#{@uri.scheme}://#{@uri.host}"
|
31
|
+
full_url += @uri.port ? ":#{@uri.port}" : ''
|
32
|
+
full_url += url
|
33
|
+
{
|
34
|
+
:status_code=>net_http_response.code.to_i,
|
35
|
+
:url=>full_url,
|
36
|
+
:body=>net_http_response.body,
|
37
|
+
:path=>path,
|
38
|
+
:params=>params,
|
39
|
+
:data=>data,
|
40
|
+
:headers=>headers
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
def _build_url(path, params={})
|
45
|
+
build_url(@uri.path + path, params, @uri.query) # build_url is coming from RSolr::HTTPClient::Util
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
@@ -0,0 +1,153 @@
|
|
1
|
+
# http://builder.rubyforge.org/
|
2
|
+
require 'rubygems'
|
3
|
+
require 'builder'
|
4
|
+
|
5
|
+
# The Solr::Message class is the XML generation module for sending updates to Solr.
|
6
|
+
|
7
|
+
class RSolr::Message
|
8
|
+
|
9
|
+
# A class that represents a "doc" xml element for a solr update
|
10
|
+
class Document
|
11
|
+
|
12
|
+
# "attrs" is a hash for setting the "doc" xml attributes
|
13
|
+
# "fields" is an array of Field objects
|
14
|
+
attr_accessor :attrs, :fields
|
15
|
+
|
16
|
+
# "doc_hash" must be a Hash/Mash object
|
17
|
+
# If a value in the "doc_hash" is an array,
|
18
|
+
# a field object is created for each value...
|
19
|
+
def initialize(doc_hash)
|
20
|
+
@fields = []
|
21
|
+
doc_hash.each_pair do |field,values|
|
22
|
+
# create a new field for each value (multi-valued)
|
23
|
+
# put non-array values into an array
|
24
|
+
values = [values] unless values.is_a?(Array)
|
25
|
+
values.each do |v|
|
26
|
+
next if v.to_s.empty?
|
27
|
+
@fields << Field.new({:name=>field}, v)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
@attrs={}
|
31
|
+
end
|
32
|
+
|
33
|
+
# returns an array of fields that match the "name" arg
|
34
|
+
def fields_by_name(name)
|
35
|
+
@fields.select{|f|f.name==name}
|
36
|
+
end
|
37
|
+
|
38
|
+
# returns the first field that matches the "name" arg
|
39
|
+
def field_by_name(name)
|
40
|
+
@fields.detect{|f|f.name==name}
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
# A class that represents a "doc"/"field" xml element for a solr update
|
46
|
+
class Field
|
47
|
+
|
48
|
+
# "attrs" is a hash for setting the "doc" xml attributes
|
49
|
+
# "value" is the text value for the node
|
50
|
+
attr_accessor :attrs, :value
|
51
|
+
|
52
|
+
# "attrs" must be a hash
|
53
|
+
# "value" should be something that responds to #_to_s
|
54
|
+
def initialize(attrs, value)
|
55
|
+
@attrs = attrs
|
56
|
+
@value = value
|
57
|
+
end
|
58
|
+
|
59
|
+
# the value of the "name" attribute
|
60
|
+
def name
|
61
|
+
@attrs[:name]
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
class << self
|
67
|
+
|
68
|
+
# shortcut method -> xml = RSolr::Message.xml
|
69
|
+
def xml
|
70
|
+
::Builder::XmlMarkup.new
|
71
|
+
end
|
72
|
+
|
73
|
+
# generates "add" xml for updating solr
|
74
|
+
# "data" can be a hash or an array of hashes.
|
75
|
+
# - each hash should be a simple key=>value pair representing a solr doc.
|
76
|
+
# If a value is an array, multiple fields will be created.
|
77
|
+
#
|
78
|
+
# "add_attrs" can be a hash for setting the add xml element attributes.
|
79
|
+
#
|
80
|
+
# This method can also accept a block.
|
81
|
+
# The value yielded to the block is a Message::Document; for each solr doc in "data".
|
82
|
+
# You can set xml element attributes for each "doc" element or individual "field" elements.
|
83
|
+
#
|
84
|
+
# For example:
|
85
|
+
#
|
86
|
+
# solr.add({:id=>1, :nickname=>'Tim'}, {:boost=>5.0, :commitWithin=>1.0}) do |doc_msg|
|
87
|
+
# doc_msg.attrs[:boost] = 10.00 # boost the document
|
88
|
+
# nickname = doc_msg.field_by_name(:nickname)
|
89
|
+
# nickname.attrs[:boost] = 20 if nickname.value=='Tim' # boost a field
|
90
|
+
# end
|
91
|
+
#
|
92
|
+
# would result in an add element having the attributes boost="10.0"
|
93
|
+
# and a commitWithin="1.0".
|
94
|
+
# Each doc element would have a boost="10.0".
|
95
|
+
# The "nickname" field would have a boost="20.0"
|
96
|
+
# if the doc had a "nickname" field with the value of "Tim".
|
97
|
+
#
|
98
|
+
def add(data, add_attrs={}, &blk)
|
99
|
+
data = [data] if data.respond_to?(:each_pair)
|
100
|
+
xml.add(add_attrs) do |add_node|
|
101
|
+
data.each do |item|
|
102
|
+
# create doc, passing in fields
|
103
|
+
doc = Document.new(item)
|
104
|
+
yield doc if block_given?
|
105
|
+
add_node.doc(doc.attrs) do |doc_node|
|
106
|
+
doc.fields.each do |field_obj|
|
107
|
+
doc_node.field(field_obj.value, field_obj.attrs)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
# generates a <commit/> message
|
115
|
+
def commit(opts={})
|
116
|
+
xml.commit(opts)
|
117
|
+
end
|
118
|
+
|
119
|
+
# generates a <optimize/> message
|
120
|
+
def optimize(opts={})
|
121
|
+
xml.optimize(opts)
|
122
|
+
end
|
123
|
+
|
124
|
+
# generates a <rollback/> message
|
125
|
+
def rollback
|
126
|
+
xml.rollback
|
127
|
+
end
|
128
|
+
|
129
|
+
# generates a <delete><id>ID</id></delete> message
|
130
|
+
# "ids" can be a single value or array of values
|
131
|
+
def delete_by_id(ids)
|
132
|
+
ids = [ids] unless ids.is_a?(Array)
|
133
|
+
xml.delete do |xml|
|
134
|
+
ids.each do |id|
|
135
|
+
xml.id(id)
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
# generates a <delete><query>ID</query></delete> message
|
141
|
+
# "queries" can be a single value or an array of values
|
142
|
+
def delete_by_query(queries)
|
143
|
+
queries = [queries] unless queries.is_a?(Array)
|
144
|
+
xml.delete do |xml|
|
145
|
+
queries.each do |query|
|
146
|
+
xml.query(query)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
end
|
152
|
+
|
153
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
if defined?(JRUBY_VERSION)
|
2
|
+
|
3
|
+
require 'helper'
|
4
|
+
require 'connection/test_methods'
|
5
|
+
|
6
|
+
class ConnectionDirectTest < RSolrBaseTest
|
7
|
+
|
8
|
+
include ConnectionTestMethods
|
9
|
+
|
10
|
+
def setup
|
11
|
+
base = File.expand_path( File.dirname(__FILE__) )
|
12
|
+
dist = File.join(base, '..', '..', 'apache-solr')
|
13
|
+
home = File.join(dist, 'example', 'solr')
|
14
|
+
@solr = RSolr.connect(:adapter=>:direct, :home_dir=>home, :dist_dir=>dist)
|
15
|
+
@solr.delete_by_query('*:*')
|
16
|
+
@solr.commit
|
17
|
+
end
|
18
|
+
|
19
|
+
def teardown
|
20
|
+
@solr.adapter.close
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
unless defined?(JRUBY_VERSION)
|
2
|
+
|
3
|
+
require 'helper'
|
4
|
+
require 'connection/test_methods'
|
5
|
+
|
6
|
+
class AdapterHTTPTest < RSolrBaseTest
|
7
|
+
|
8
|
+
include ConnectionTestMethods
|
9
|
+
|
10
|
+
def setup
|
11
|
+
@solr = RSolr.connect
|
12
|
+
@solr.delete_by_query('*:*')
|
13
|
+
@solr.commit
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|