rsolr 0.9.6 → 0.9.7.1

data/CHANGES.txt

@@ -1,3 +1,18 @@
+0.9.7.1 - November 5, 2009
+  added support for multibyte url characters in RSolr::Connection::Utils
+    - this is because Ruby 1.9's String size method is different than 1.8
+
+0.9.7 - October 23, 2009
+  Removed XML message builders - using pure Ruby generator instead
+    - benchmarks show generation speed is a little faster than libxml
+    - minimal xml escaping so binary posting to Solr should no longer be a problem.
+  Changed response.adapter_resonse to response.raw
+  Removed HTTP adapters - sticking with NetHTTP
+  Removed builder dependency in gemspec
+  Removed Adapter and HTTPClient modules
+  Moved HTTPCLient::Util to Connection::Utils
+  Updated all tests
+
 0.9.6 - September 9, 2009
   Added ability to create direct connections from existing Java::OrgApacheSolrCore::SolrCore
   Added ability to send queries using POST

data/README.rdoc

@@ -3,8 +3,8 @@
 A Ruby client for Apache Solr. RSolr has been developed to be simple and extendable. It features transparent JRuby DirectSolrConnection support and a simple Hash-in, Hash-out architecture.
 
 == Installation:
-  gem sources -a http://gems.github.com
-  sudo gem install mwmitchell-rsolr
+  gem sources -a http://gemcutter.org
+  sudo gem install rsolr
 
 ==Related Resources & Projects
 * {Solr}[http://lucene.apache.org/solr/]
@@ -100,12 +100,6 @@ Commit & optimize shortcuts
   solr.commit
   solr.optimize
 
-===XML Builders for RSolr
-As of version 0.9.1, RSolr can use LibXml to create the update messages sent to solr. To switch from Builder to LibXml, set the RSolr::Message.builder like:
-  solr = RSolr.connect
-  solr.message.adapter = RSolr::Message::Adapter::Libxml.new
-
-
 == Response Formats
 The default response format is Ruby. When the :wt param is set to :ruby, the response is eval'd resulting in a Hash. You can get a raw response by setting the :wt to "ruby" - notice, the string -- not a symbol. RSolr will eval the Ruby string ONLY if the :wt value is :ruby. All other response formats are available as expected, :wt=>'xml' etc..
 
@@ -119,30 +113,10 @@ The default response format is Ruby. When the :wt param is set to :ruby, the res
 ===JSON:
   solr.select(:wt=>:json)
 
-You can access the original request context (path, params, url etc.) by calling the #adapter_response method:
+You can access the original request context (path, params, url etc.) by calling the #raw method:
   response = solr.select :q=>'*:*'
-  response.adapter_response[:status_code]
-  response.adapter_response[:body]
-  response.adapter_response[:url]
-
-The adapter_response is a hash that contains the generated params, url, path, post data, headers etc., very useful for debugging and testing.
-
-
-== HTTP Client Adapter
-You can specify the http client adapter:
-  :net_http     uses the standard Net::HTTP library
-  :curb         uses the C based "curl" library
+  response.raw[:status_code]
+  response.raw[:body]
+  response.raw[:url]
 
-NOTE: The Net::Http is the default adapter.
-
-Example:
-  
-  RSolr.connect(:adapter => :curb)
-  RSolr.connect(:adapter => :net_http)
-
-Intereseting read about Ruby's Net::HTTP library:
-http://apocryph.org/2008/11/09/more_indepth_analysis_ruby_http_client_performance
-
-NOTE: You can't use the :curb adapter under jRuby. To install curb:
-  
-  sudo gem install curb
+The raw is a hash that contains the generated params, url, path, post data, headers etc., very useful for debugging and testing.

data/Rakefile

@@ -1,76 +1,14 @@
 require 'rake'
 require 'rake/testtask'
 require 'rake/rdoctask'
-
-namespace :rsolr do
-  
-  desc "Starts the HTTP server used for running HTTP connection tests"
-  task :start_test_server do
-    system "cd apache-solr/example; java -jar start.jar"
-  end
-  
-end
-
-task :default => [:test_units]
-
-# rake package
-
-require 'rubygems'
 require 'rake/gempackagetask'
-raw_spec = File.read 'rsolr.gemspec'
-spec = eval(raw_spec)
-Rake::GemPackageTask.new(spec) do |pkg|
-    pkg.need_tar = true
-end
-
-desc "Run basic tests"
-Rake::TestTask.new("test_units") { |t|
-  t.pattern = 'test/**/*_test.rb'
-  t.verbose = true
-  t.warning = true
-  t.libs << "test"
-}
-
-require 'spec/rake/spectask'
-
-desc "Run specs"
-Spec::Rake::SpecTask.new('spec') do |t|
-  t.spec_files = FileList['spec/**/*_spec.rb']
-  t.libs += ["lib", "spec"]
-end
-
-desc 'Run specs' # this task runs each test in its own process
-task :specs do
-  require 'rubygems'
-  require 'facets/more/filelist' unless defined?(FileList)
-  files = FileList["**/*_spec.rb"]
-  p files.to_a
-  files.each do |filename|
-    system "cd #{File.dirname(filename)} && ruby #{File.basename(filename)}"
-  end
-end
-
-desc "Run specs"
-Rake::TestTask.new("specs") { |t|
-  t.pattern = 'spec/**/*_spec.rb'
-  t.verbose = true
-  t.warning = true
-  t.libs += ["lib", "spec"]
-}
 
-# Clean house
-desc 'Clean up tmp files.'
-task :clean do |t|
-  FileUtils.rm_rf "doc"
-  FileUtils.rm_rf "pkg"
+ENV['RUBYOPT'] = '-W1'
+ 
+task :environment do
+  require File.dirname(__FILE__) + '/lib/rsolr'
 end
+ 
+Dir['tasks/**/*.rake'].each { |t| load t }
 
-# Rdoc
-desc 'Generate documentation for the rsolr gem.'
-Rake::RDocTask.new(:doc) do |rdoc|
-  rdoc.rdoc_dir = 'doc'
-  rdoc.title = 'RSolr'
-  rdoc.options << '--line-numbers' << '--inline-source'
-  rdoc.rdoc_files.include('README.rdoc')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
+task :default => ['spec:api']

data/examples/http.rb

@@ -1,7 +1,6 @@
 require File.join(File.dirname(__FILE__), '..', 'lib', 'rsolr')
 
-# switch out the http adapter from net_http to curb
-solr = RSolr.connect :adapter=>:curb
+solr = RSolr.connect
 
 Dir['../apache-solr/example/exampledocs/*.xml'].each do |xml_file|
   puts "Updating with #{xml_file}"
@@ -14,7 +13,7 @@ puts
 
 response = solr.select(:q=>'ipod', :fq=>['price:[0 TO 50]'], :rows=>2, :start=>0)
 
-puts "URL : #{response.adapter_response[:url]} -> #{response.adapter_response[:status_code]}"
+puts "URL : #{response.raw[:url]} -> #{response.raw[:status_code]}"
 
 puts
 

data/lib/rsolr/connection/{adapter/direct.rb → direct.rb}

@@ -5,9 +5,9 @@ require 'java'
 #
 # Connection for JRuby + DirectSolrConnection
 #
-class RSolr::Connection::Adapter::Direct
+class RSolr::Connection::Direct
   
-  include RSolr::HTTPClient::Util
+  include RSolr::Connection::Utils
   
   attr_accessor :opts
   

data/lib/rsolr/connection/net_http.rb

@@ -0,0 +1,79 @@
+require 'net/http'
+
+#
+# Connection for standard HTTP Solr server
+#
+class RSolr::Connection::NetHttp
+  
+  include RSolr::Connection::Utils
+  
+  attr_reader :opts, :uri
+  
+  # opts can have:
+  #   :url => 'http://localhost:8080/solr'
+  def initialize opts={}
+    opts[:url] ||= 'http://127.0.0.1:8983/solr'
+    @opts = opts
+    @uri = URI.parse opts[:url]
+  end
+  
+  # send a request to the connection
+  # request '/update', :wt=>:xml, '</commit>'
+  def request path, params={}, *extra
+    opts = extra[-1].kind_of?(Hash) ? extra.pop : {}
+    data = extra[0]
+    # force a POST, use the query string as the POST body
+    if opts[:method] == :post and data.to_s.empty?
+      http_context = self.post(path, hash_to_query(params), {}, {'Content-Type' => 'application/x-www-form-urlencoded'})
+    else
+      if data
+        # standard POST, using "data" as the POST body
+        http_context = self.post(path, data, params, {"Content-Type" => 'text/xml; charset=utf-8'})
+      else
+        # standard GET
+        http_context = self.get(path, params)
+      end
+    end
+    raise RSolr::RequestError.new(http_context[:body]) unless http_context[:status_code] == 200
+    http_context
+  end
+  
+  protected
+  
+  def connection
+    @connection ||= Net::HTTP.new(@uri.host, @uri.port)
+  end
+  
+  def get path, params={}
+    url = self.build_url path, params
+    net_http_response = self.connection.get url
+    create_http_context net_http_response, url, path, params
+  end
+  
+  def post path, data, params={}, headers={}
+    url = self.build_url path, params
+    net_http_response = self.connection.post url, data, headers
+    create_http_context net_http_response, url, path, params, data, headers
+  end
+  
+  def create_http_context net_http_response, url, path, params, data=nil, headers={}
+    full_url = "#{@uri.scheme}://#{@uri.host}"
+    full_url += @uri.port ? ":#{@uri.port}" : ''
+    full_url += url
+    {
+      :status_code=>net_http_response.code.to_i,
+      :url=>full_url,
+      :body=>net_http_response.body,
+      :path=>path,
+      :params=>params,
+      :data=>data,
+      :headers=>headers
+    }
+  end
+  
+  def build_url path, params={}
+    full_path = @uri.path + path
+    super full_path, params, @uri.query
+  end
+  
+end

data/lib/rsolr/connection.rb

@@ -1,124 +1,72 @@
 module RSolr::Connection
   
-  module Adapter
-    autoload :Direct, 'rsolr/connection/adapter/direct'
-    autoload :HTTP, 'rsolr/connection/adapter/http'
-  end
-  
-  class Base
-    
-    attr_reader :adapter
-    
-    # "adapter" is instance of:
-    #   RSolr::Adapter::HTTP
-    #   RSolr::Adapter::Direct (jRuby only)
-    # or any other class that uses the connection "interface"
-    def initialize(adapter)
-      @adapter = adapter
-    end
+  autoload :Direct, 'rsolr/connection/direct'
+  autoload :NetHttp, 'rsolr/connection/net_http'
   
-    # Send a request to a request handler using the method name.
-    def method_missing(method_name, *args, &blk)
-      request("/#{method_name}", *args, &blk)
+  # Helpful utility methods for building queries to a Solr server
+  module Utils
+
+    # Performs URI escaping so that you can construct proper
+    # query strings faster.  Use this rather than the cgi.rb
+    # version since it's faster.  (Stolen from Rack).
+    def escape(s)
+      s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
+        #'%'+$1.unpack('H2'*$1.size).join('%').upcase
+        '%'+$1.unpack('H2'*bytesize($1)).join('%').upcase
+      }.tr(' ', '+')
     end
     
-    # sends data to the update handler
-    # data can be a string of xml, or an object that returns xml from its #to_xml method
-    def update(data, params={})
-      request '/update', params, data
+    # Return the bytesize of String; uses String#length under Ruby 1.8 and
+    # String#bytesize under 1.9.
+    if ''.respond_to?(:bytesize)
+      def bytesize(string)
+        string.bytesize
+      end
+    else
+      def bytesize(string)
+        string.size
+      end
     end
     
-    # send request solr
-    # params is hash with valid solr request params (:q, :fl, :qf etc..)
-    #   if params[:wt] is not set, the default is :ruby
-    #   if :wt is something other than :ruby, the raw response body is used
-    #   otherwise, a simple Hash is returned
-    #   NOTE: to get raw ruby, use :wt=>'ruby' <- a string, not a symbol like :ruby  
-    #
-    #
-    def request(path, params={}, *extra)
-      response = @adapter.request(path, map_params(params), *extra)
-      adapt_response(response)
+    # creates and returns a url as a string
+    # "url" is the base url
+    # "params" is an optional hash of GET style query params
+    # "string_query" is an extra query string that will be appended to the 
+    # result of "url" and "params".
+    def build_url url='', params={}, string_query=''
+      queries = [string_query, hash_to_query(params)]
+      queries.delete_if{|i| i.to_s.empty?}
+      url += "?#{queries.join('&')}" unless queries.empty?
+      url
     end
-    
-    # 
-    # single record:
-    # solr.update(:id=>1, :name=>'one')
+
+    # converts a key value pair to an escaped string:
+    # Example:
+    # build_param(:id, 1) == "id=1"
+    def build_param(k,v)
+      "#{escape(k)}=#{escape(v)}"
+    end
+
     #
-    # update using an array
-    # solr.update([{:id=>1, :name=>'one'}, {:id=>2, :name=>'two'}])
+    # converts hash into URL query string, keys get an alpha sort
+    # if a value is an array, the array values get mapped to the same key:
+    #   hash_to_query(:q=>'blah', :fq=>['blah', 'blah'], :facet=>{:field=>['location_facet', 'format_facet']})
+    # returns:
+    #   ?q=blah&fq=blah&fq=blah&facet.field=location_facet&facet.field=format.facet
     #
-    def add(doc, &block)
-      update message.add(doc, &block)
-    end
-  
-    # send </commit>
-    def commit
-      update message.commit
-    end
-  
-    # send </optimize>
-    def optimize
-      update message.optimize
-    end
-  
-    # send </rollback>
-    # NOTE: solr 1.4 only
-    def rollback
-      update message.rollback
-    end
-  
-    # Delete one or many documents by id
-    #   solr.delete_by_id 10
-    #   solr.delete_by_id([12, 41, 199])
-    def delete_by_id(id)
-      update message.delete_by_id(id)
-    end
-  
-    # delete one or many documents by query
-    #   solr.delete_by_query 'available:0'
-    #   solr.delete_by_query ['quantity:0', 'manu:"FQ"']
-    def delete_by_query(query)
-      update message.delete_by_query(query)
-    end
-    
-    # shortcut to RSolr::Message::Builder
-    def message
-      @message ||= RSolr::Message::Builder.new
-    end
-    
-    protected
-    
-    # sets default params etc.. - could be used as a mapping hook
-    # type of request should be passed in here? -> map_params(:query, {})
-    def map_params(params)
-      params||={}
-      {:wt=>:ruby}.merge(params)
-    end
-  
-    # "adapter_response" must be a hash with the following keys:
-    #   :params - a sub hash of standard solr params
-    #   : body - the raw response body from the solr server
-    # This method will evaluate the :body value if the params[:wt] == :ruby
-    # otherwise, the body is returned
-    # The return object has a special method attached called #adapter_response
-    # This method gives you access to the original response from the adapter,
-    # so you can access things like the actual :url sent to solr,
-    # the raw :body, original :params and original :data
-    def adapt_response(adapter_response)
-      data = adapter_response[:body]
-      # if the wt is :ruby, evaluate the ruby string response
-      if adapter_response[:params][:wt] == :ruby
-        data = Kernel.eval(data)
-      end
-      # attach a method called #adapter_response that returns the original adapter response value
-      def data.adapter_response
-        @adapter_response
+    # if a value is empty/nil etc., it is not added
+    def hash_to_query(params)
+      mapped = params.map do |k, v|
+        next if v.to_s.empty?
+        if v.class == Array
+          hash_to_query(v.map { |x| [k, x] })
+        else
+          build_param k, v
+        end
       end
-      data.send(:instance_variable_set, '@adapter_response', adapter_response)
-      data
+      mapped.compact.join("&")
     end
-    
+
   end
   
 end

data/lib/rsolr/message.rb

@@ -2,11 +2,6 @@
 
 module RSolr::Message
   
-  module Adapter
-    autoload :Builder, 'rsolr/message/adapter/builder'
-    autoload :Libxml, 'rsolr/message/adapter/libxml'
-  end
-  
   # A class that represents a "doc" xml element for a solr update
   class Document
     
@@ -25,7 +20,7 @@ module RSolr::Message
         values = [values] unless values.is_a?(Array)
         values.each do |v|
           next if v.to_s.empty?
-          @fields << Field.new({:name=>field}, v)
+          @fields << Field.new({:name=>field}, v.to_s)
         end
       end
       @attrs={}
@@ -79,15 +74,6 @@ module RSolr::Message
   
   class Builder
     
-    attr_writer :adapter
-    
-    # b = Builder.new
-    # b.adapter = RSolr::Message::Adapter::LibXML.new
-    # b.optimize == '<optimize/>'
-    def adapter
-      @adapter ||= RSolr::Message::Adapter::Builder.new
-    end
-    
     # generates "add" xml for updating solr
     # "data" can be a hash or an array of hashes.
     # - each hash should be a simple key=>value pair representing a solr doc.
@@ -115,39 +101,54 @@ module RSolr::Message
     #
     def add(data, add_attrs={})
       data = [data] unless data.is_a?(Array)
-      documents = data.map do |doc|
+      add = Xout.new :add, add_attrs
+      data.each do |doc|
         doc = Document.new(doc) if doc.respond_to?(:each_pair)
         yield doc if block_given?
-        doc
+        add.child :doc, doc.attrs do |doc_node|
+          doc.fields.each do |field_obj|
+            doc_node.child :field, field_obj.value, field_obj.attrs
+          end
+        end
       end
-      adapter.add(documents, add_attrs)
+      add.to_xml
     end
     
     # generates a <commit/> message
     def commit(opts={})
-      adapter.commit(opts)
+      Xout.new(:commit, opts).to_xml
     end
     
     # generates a <optimize/> message
     def optimize(opts={})
-      adapter.optimize(opts)
+      Xout.new(:optimize, opts).to_xml
     end
     
     # generates a <rollback/> message
     def rollback
-      adapter.rollback
+      Xout.new(:rollback).to_xml
     end
     
     # generates a <delete><id>ID</id></delete> message
     # "ids" can be a single value or array of values
     def delete_by_id(ids)
-      adapter.delete_by_id(ids)
+      ids = [ids] unless ids.is_a?(Array)
+      delete_node = Xout.new(:delete) do |xml|
+        ids.each { |id| xml.child :id, id }
+      end
+      delete_node.to_xml
     end
     
     # generates a <delete><query>ID</query></delete> message
     # "queries" can be a single value or an array of values
     def delete_by_query(queries)
-      adapter.delete_by_query(queries)
+      queries = [queries] unless queries.is_a?(Array)
+      delete_node = Xout.new(:delete) do |xml|
+        queries.each { |query| xml.child :query, query }
+      end
+      delete_node.to_xml
     end
+    
   end
-end
+  
+end

data/lib/rsolr.rb

@@ -4,13 +4,15 @@ require 'rubygems'
 
 $: << File.dirname(__FILE__) unless $:.include?(File.dirname(__FILE__))
 
+require 'xout'
+
 module RSolr
   
-  VERSION = '0.9.5'
+  VERSION = '0.9.7'
   
   autoload :Message, 'rsolr/message'
+  autoload :Client, 'rsolr/client'
   autoload :Connection, 'rsolr/connection'
-  autoload :HTTPClient, 'rsolr/http_client'
   
   # Factory for creating connections.
   # 2 modes of argument operations:
@@ -28,16 +30,16 @@ module RSolr
     type = args.first.is_a?(Symbol) ? args.shift : :http
     opts = args
     type_class = case type
-      when :http,nil
-        'HTTP'
+      when :net_http,:http,nil
+        'NetHttp'
       when :direct
         'Direct'
       else
         raise "Invalid connection type: #{type} - use :http, :direct or leave nil for :http/default"
       end
-    adapter_class = RSolr::Connection::Adapter.const_get(type_class)
+    adapter_class = RSolr::Connection.const_get(type_class)
     adapter = adapter_class.new(*opts)
-    RSolr::Connection::Base.new(adapter)
+    RSolr::Client.new(adapter)
   end
   
   # A module that contains string related methods
@@ -55,7 +57,7 @@ module RSolr
   # send the escape method into the Connection class ->
   # solr = RSolr.connect
   # solr.escape('asdf')
-  RSolr::Connection::Base.send(:include, Char)
+  RSolr::Client.send(:include, Char)
   
   # bring escape into this module (RSolr) -> RSolr.escape('asdf')
   extend Char

data/lib/xout.rb

@@ -0,0 +1,65 @@
+class Xout
+  
+  attr_reader :name, :text, :attrs, :children
+  
+  def initialize node_name, *args, &block
+    @children = []
+    attrs = args.last.is_a?(Hash) ? args.pop : {}
+    text = args.empty? ? '' : args.pop.to_s
+    @name, @text, @attrs = node_name, text, attrs
+    yield self if block_given?
+  end
+  
+  def child name, *args, &block
+    add_child self.class.new(name, *args, &block)
+  end
+  
+  def add_child node
+    children << node
+  end
+  
+  def to_xml
+    xml = ["<#{name}#{create_attrs(attrs)}"]
+    if not text.empty? or not children.empty?
+      xml << ">#{escape_text text.to_s}"
+      xml += children.map{|child|child.to_xml}
+      xml << "</#{name}>"
+    else
+      xml << '/>'
+    end
+    xml.join
+  end
+  
+  alias :to_s :to_xml
+  
+  def to_xml_doc
+    '<?xml version="1.0" encoding="UTF-8"?>' + to_xml
+  end
+  
+  protected
+  
+  # builds an XML attribute string.
+  # escapes each attribute value by running it through #escape_attr
+  def create_attrs hash
+    r = hash.map { |k,v| "#{k}=\"#{escape_attr v.to_s}\"" }.join(' ')
+    " #{r}" unless r.empty?
+  end
+  
+  # minimal escaping for attribute values
+  def escape_attr input
+    escape input, '&'=>'&amp;', '<'=>'&lt;', '>'=>'&gt;', "'"=>'&apos;', '"'=>'&quote;'
+  end
+  
+  # minimal escaping for text
+  def escape_text input
+    escape input, '&'=>'&amp;', '<'=>'&lt;', '>'=>'&gt;'
+  end
+  
+  # accepts a string input and a hash mapping of characters => replacement values:
+  # Example:
+  #   escape 'My <string>cat</strong>', '<'=>'&gt;', '>'=>'&lt;'
+  def escape input, map
+    input.gsub(/[#{map.keys.join}]+/) { | char | map[char] || char }
+  end
+  
+end