mwmitchell-rsolr 0.5.7

data/lib/rsolr/http_client.rb

@@ -0,0 +1,115 @@
+#require 'uri'
+
+# A simple wrapper for different http client implementations.
+# Supports #get and #post
+# This was motivated by: http://apocryph.org/2008/11/09/more_indepth_analysis_ruby_http_client_performance/
+# Net::HTTP is the default adapter
+
+# Each adapter response should be a hash with the following keys:
+#   :status_code
+#   :url
+#   :body
+#   :path
+#   :params
+#   :data
+#   :headers
+
+# Example:
+#   hclient = RSolr::HTTPClient.connect('http://www.google.com', :net_http)
+#   response = hclient.get('/search', :hl=>:en, :q=>:ruby, :btnG=>:Search)
+#   puts response[:status_code]
+#   puts response[:body]
+
+module RSolr::HTTPClient
+  
+  autoload :Adapter, 'rsolr/http_client/adapter'
+  
+  class UnkownAdapterError < RuntimeError; end
+  
+  def self.connect(url, adapter_name=:net_http)
+    case adapter_name
+    when :curb
+      klass = 'Curb'
+    when :net_http
+      klass = 'NetHTTP'
+    else
+      raise UnkownAdapterError.new("Name: #{adapter_name}")
+    end
+    Base.new RSolr::HTTPClient::Adapter.const_get(klass).new(url)
+  end
+  
+  class Base
+    
+    attr_reader :adapter
+    
+    def initialize(adapter)
+      @adapter = adapter
+    end
+    
+    def get(path, params={})
+      begin
+        http_context = @adapter.get(path, params)
+      rescue
+        raise RSolr::RequestError.new($!)
+      end
+      http_context
+    end
+    
+    def post(path, data, params={}, headers={})
+      begin
+        http_context = @adapter.post(path, data, params, headers)
+      rescue
+        raise RSolr::RequestError.new($!)
+      end
+      http_context
+    end
+    
+  end
+  
+  module Util
+    
+    # escapes a query key/value for http
+    def escape(s)
+      s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
+        '%'+$1.unpack('H2'*$1.size).join('%').upcase
+      }.tr(' ', '+') 
+    end
+
+    def build_url(url='', params={}, string_query='')
+      queries = [string_query, hash_to_params(params)]
+      queries.delete_if{|i| i.to_s.empty?}
+      url += "?#{queries.join('&')}" unless queries.empty?
+      url
+    end
+
+    def build_param(k,v)
+      "#{escape(k)}=#{escape(v)}"
+    end
+
+    #
+    # converts hash into URL query string, keys get an alpha sort
+    # if a value is an array, the array values get mapped to the same key:
+    #   hash_to_params(:q=>'blah', 'facet.field'=>['location_facet', 'format_facet'])
+    # returns:
+    #   ?q=blah&facet.field=location_facet&facet.field=format.facet
+    #
+    # if a value is empty/nil etc., the key is not added
+    def hash_to_params(params)
+      return unless params.is_a?(Hash)
+      # copy params and convert keys to strings
+      params = params.inject({}){|acc,(k,v)| acc.merge({k.to_s, v}) }
+      # get sorted keys
+      params.keys.sort.inject([]) do |acc,k|
+        v = params[k]
+        if v.is_a?(Array)
+          acc << v.reject{|i|i.to_s.empty?}.collect{|vv|build_param(k, vv)}
+        elsif ! v.to_s.empty?
+          acc.push(build_param(k, v))
+        end
+        acc
+      end.join('&')
+    end
+    
+  end
+  
+end

data/lib/rsolr/http_client/adapter.rb

@@ -0,0 +1,6 @@
+module RSolr::HTTPClient::Adapter
+  
+  autoload :Curb, 'rsolr/http_client/adapter/curb'
+  autoload :NetHTTP, 'rsolr/http_client/adapter/net_http'
+  
+end

data/lib/rsolr/http_client/adapter/curb.rb

@@ -0,0 +1,51 @@
+require 'rubygems'
+require 'curb'
+
+class RSolr::HTTPClient::Adapter::Curb
+  
+  include RSolr::HTTPClient::Util
+  
+  attr :uri
+  attr :c
+  
+  def initialize(url)
+    @uri = URI.parse(url)
+    @c = ::Curl::Easy.new
+  end
+  
+  def get(path, params={})
+    @c.url = _build_url(path, params)
+    @c.multipart_form_post = false
+    @c.perform
+    create_http_context(path, params)
+  end
+  
+  def post(path, data, params={}, headers={})
+    @c.url = _build_url(path, params)
+    @c.headers = headers
+    @c.http_post(data)
+    create_http_context(path, params, data, headers)
+  end
+  
+  protected
+  
+  def create_http_context(path, params, data=nil, headers={})
+    {
+      :status_code=>@c.response_code.to_i,
+      :url=>@c.url,
+      :body=>@c.body_str,
+      :path=>path,
+      :params=>params,
+      :data=>data,
+      :headers=>headers
+    }
+  end
+  
+  def _build_url(path, params={})
+    url = @uri.scheme + '://' + @uri.host
+    url += ':' + @uri.port.to_s if @uri.port
+    url += @uri.path + path
+    build_url(url, params, @uri.query)
+  end
+  
+end

data/lib/rsolr/http_client/adapter/net_http.rb

@@ -0,0 +1,48 @@
+require 'net/http'
+
+class RSolr::HTTPClient::Adapter::NetHTTP
+  
+  include RSolr::HTTPClient::Util
+  
+  attr :uri
+  attr :c
+  
+  def initialize(url)
+    @uri = URI.parse(url)
+    @c = Net::HTTP.new(@uri.host, @uri.port)
+  end
+  
+  def get(path, params={})
+    url = _build_url(path, params)
+    net_http_response = @c.get(url)
+    create_http_context(net_http_response, url, path, params)
+  end
+  
+  def post(path, data, params={}, headers={})
+    url = _build_url(path, params)
+    net_http_response = @c.post(url, data, headers)
+    create_http_context(net_http_response, url, path, params, data, headers)
+  end
+  
+  protected
+  
+  def create_http_context(net_http_response, url, path, params, data=nil, headers={})
+    full_url = "#{@uri.scheme}://#{@uri.host}"
+    full_url += @uri.port ? ":#{@uri.port}" : ''
+    full_url += url
+    {
+      :status_code=>net_http_response.code.to_i,
+      :body=>net_http_response.body,
+      :url=>full_url,
+      :path=>path,
+      :params=>params,
+      :data=>data,
+      :headers=>headers
+    }
+  end
+  
+  def _build_url(path, params={})
+    build_url(@uri.path + path, params, @uri.query)
+  end
+  
+end

data/lib/rsolr/indexer.rb

@@ -0,0 +1,23 @@
+class RSolr::Indexer
+  
+  attr_reader :solr, :mapper, :opts
+  
+  def initialize(solr, mapping_or_mapper, opts={})
+    @solr = solr
+    @mapper = mapping_or_mapper.is_a?(Hash) ? RSolr::Mapper::Base.new(mapping_or_mapper) : mapping_or_mapper
+    @opts = opts
+  end
+  
+  # data - the raw data to send into the mapper
+  # params - url query params for solr /update handler
+  # commit - boolean; true==commit after adding, false==no commit after adding
+  # block can be used for modifying the "add", "doc" and "field" xml elements (for boosting etc.)
+  def index(data, params={}, &block)
+    docs = data.collect {|d| @mapper.map(d)}
+    @solr.add(docs, params) do |add, doc, field|
+      # check opts for :debug etc.?
+      yield add, doc, field if block_given?
+    end
+  end
+  
+end

data/lib/rsolr/mapper.rb

@@ -0,0 +1,62 @@
+module RSolr::Mapper
+  
+  autoload :RSS, 'rsolr/mapper/rss'
+  
+  class UnkownMappingValue < RuntimeError; end
+  
+  class Base
+    
+    attr_reader :mapping, :opts
+  
+    def initialize(mapping={}, opts={}, &block)
+      @mapping = mapping
+      @opts = opts
+      yield @mapping if block_given?
+    end
+    
+    # source - a hash or array of source data
+    # override_mapping - an alternate mapper
+    # returns an array with one or more mapped hashes
+    def map(source, override_mapping=nil, &block)
+      source = [source] if source.is_a?(Hash)
+      mapping = override_mapping || @mapping
+      index = -1
+      # collect a bunch of hashes...
+      source.collect do |src|
+        index += 1
+        # for each mapping item, inject data into a new hash
+        final_hash = mapping.inject({}) do |a_new_hash, (map_key, map_value)|
+          value = mapped_field_value(src, map_value, index)
+          value.to_s.empty? ? a_new_hash : a_new_hash.merge!({map_key=>value})
+        end
+        yield final_hash if block_given?
+        final_hash
+      end
+    end
+  
+    protected
+  
+    # This is a hook method useful for subclassing
+    def source_field_value(source, field_name, index)
+      source[field_name]
+    end
+  
+    def mapped_field_value(source, mapped_value, index)
+      case mapped_value
+        when String
+          mapped_value
+        when Symbol
+          source_field_value(source, mapped_value, index)
+        when Proc
+          mapped_value.call(source, index)
+        when Enumerable
+          mapped_value.collect {|key| source_field_value(source, key, index)}.flatten
+        else
+          # try to turn it into a string, else raise UnkownMappingValue
+          mapped_value.respond_to?(:to_s) ? mapped_value.to_s : raise(UnkownMappingValue.new(mapped_value))
+      end
+    end
+    
+  end
+  
+end

data/lib/rsolr/mapper/rss.rb

@@ -0,0 +1,29 @@
+#TODO - this could use the http wrapper stuff instead of open-uri/net::http
+
+require 'rss'
+require 'open-uri'
+
+class RSolr::Mapper::RSS < RSolr::Mapper::Base
+  
+  attr_reader :rss
+  
+  # rss_file_or_url is file path or url (see open-uri)
+  # override_mapping is an alternate mapping (see Solr::Mapper::Base)
+  # returns array of mapped hashes
+  def map(rss_file_or_url, override_mapping=nil)
+    open(rss_file_or_url) do |feed|
+       @rss = RSS::Parser.parse(feed.read, false)
+       super(rss.items.collect, override_mapping)
+    end
+  end
+  
+  # sends methods chain down into the @rss object
+  # example: :'channel.title' == @rss.channel.title
+  # if the method chain doesn't exist, the super #source_field_value method is called
+  def source_field_value(source, method_path, index)
+    method_path.to_s.split('.').inject(@rss) do |rss, m|
+      rss.respond_to?(m) ? rss.send(m.to_sym) : super(source, method_path, index)
+    end
+  end
+  
+end

data/lib/rsolr/message.rb

@@ -0,0 +1,73 @@
+
+# http://builder.rubyforge.org/
+require 'rubygems'
+require 'builder'
+
+# The Solr::Message class is the XML generation module for sending updates to Solr.
+
+class RSolr::Message
+  
+  class << self
+    
+    def xml
+      Builder::XmlMarkup.new
+    end
+    
+    # add({})
+    # add([{}, {}])
+    # add(docs) do |doc|
+    #   doc.boost = 10.0
+    # end
+    def add(data, opts={}, &block)
+      data = [data] if data.respond_to?(:each_pair) # if it's a hash, put it in an array
+      xml.add(opts) do |add_xml|
+        data.each do |item|
+          add_xml.doc do |doc_xml|
+            # convert keys into strings and perform an alpha sort (easier testing between ruby and jruby)
+            # but probably not great for performance? whatever...
+            sorted_items = item.inject({}) {|acc,(k,v)| acc.merge({k.to_s=>v})}
+            sorted_items.keys.sort.each do |k|
+              doc_attrs = {:name=>k}
+              yield item, doc_attrs if block_given?
+              [sorted_items[k]].flatten.each do |v| # multiValued attributes
+                doc_xml.field(v, doc_attrs)
+              end
+            end
+          end
+        end
+      end
+    end
+    
+    def commit(opts={})
+      xml.commit(opts)
+    end
+    
+    def optimize(opts={})
+      xml.optimize(opts)
+    end
+    
+    def rollback
+      xml.rollback
+    end
+    
+    def delete_by_id(ids)
+      ids = [ids] unless ids.is_a?(Array)
+      xml.delete do |xml|
+        ids.each do |id|
+          xml.id(id)
+        end
+      end
+    end
+    
+    def delete_by_query(queries)
+      queries = [queries] unless queries.is_a?(Array)
+      xml.delete do |xml|
+        queries.each do |query|
+          xml.query(query)
+        end
+      end
+    end
+    
+  end
+  
+end

data/lib/rsolr/response.rb

@@ -0,0 +1,8 @@
+module RSolr::Response
+  
+  autoload :Base, 'rsolr/response/base'
+  autoload :Query, 'rsolr/response/query'
+  autoload :IndexInfo, 'rsolr/response/index_info'
+  autoload :Update, 'rsolr/response/update'
+  
+end

data/lib/rsolr/response/base.rb

@@ -0,0 +1,33 @@
+# default/base response object
+# This is where the ruby "eval" happens
+# So far, all response classes extend this
+class RSolr::Response::Base
+  
+  attr_reader :source
+  
+  attr_reader :raw_response, :data, :header, :params, :status, :query_time
+  
+  def initialize(data)
+    if data.is_a?(Hash) and data.has_key?(:body)
+      @raw_response = data[:body]
+      @data = Kernel.eval(@raw_response)
+      @source = data
+    else
+      if data.is_a?(String)
+        @raw_response = data
+        @data = Kernel.eval(@raw_response)
+      elsif data.is_a?(Hash)
+        @data = data
+      end
+    end
+    @header = @data['responseHeader']
+    @params = @header['params']
+    @status = @header['status']
+    @query_time = @header['QTime']
+  end
+  
+  def ok?
+    self.status==0
+  end
+  
+end