solr-ruby 0.0.1 → 0.0.2

data/lib/solr/connection.rb

@@ -12,6 +12,8 @@
 
 require 'net/http'
 
+# TODO: add a convenience method to POST a Solr .xml file, like Solr's example post.sh
+
 class Solr::Connection
   attr_reader :url, :autocommit, :connection
 
@@ -40,6 +42,8 @@ class Solr::Connection
   
     # Not actually opening the connection yet, just setting up the persistent connection.
     @connection = Net::HTTP.new(@url.host, @url.port)
+    
+    @connection.read_timeout = opts[:timeout] if opts[:timeout]
   end
 
   # add a document to the index. you can pass in either a hash
@@ -85,8 +89,8 @@ class Solr::Connection
   end
 
   # sends a commit message to the server
-  def commit
-    response = send(Solr::Request::Commit.new)
+  def commit(options={})
+    response = send(Solr::Request::Commit.new(options))
     return response.ok?
   end
 
@@ -136,7 +140,7 @@ class Solr::Connection
   def post(request)
     response = @connection.post(@url.path + "/" + request.handler,
                                 request.to_s,
-                                { "Content-Type" => "application/x-www-form-urlencoded; charset=utf-8" })
+                                { "Content-Type" => request.content_type })
   
     case response
     when Net::HTTPSuccess then response.body

data/lib/solr/document.rb

@@ -15,6 +15,7 @@ require 'solr/field'
 
 class Solr::Document
   include Enumerable
+  attr_accessor :boost
 
   # Create a new Solr::Document, optionally passing in a hash of 
   # key/value pairs for the fields
@@ -65,6 +66,7 @@ class Solr::Document
   # convert the Document to a REXML::Element 
   def to_xml
     e = Solr::XML::Element.new 'doc'
+    e.attributes['boost'] = @boost.to_s if @boost
     @fields.each {|f| e.add_element(f.to_xml)}
     return e
   end

data/lib/solr/field.rb

@@ -14,13 +14,16 @@ require 'solr/xml'
 require 'time'
 
 class Solr::Field
+  VALID_PARAMS = [:boost]
   attr_accessor :name
   attr_accessor :value
+  attr_accessor :boost
 
-  def initialize(key_val, opts={})
-    raise "first argument must be a hash" unless key_val.kind_of? Hash
-    @name = key_val.keys[0].to_s
-    @value = key_val.values[0]
+  # Accepts an optional <tt>:boost</tt> parameter, used to boost the relevance of a particular field.
+  def initialize(params)
+    @boost = params[:boost]
+    name_key = (params.keys - VALID_PARAMS).first
+    @name, @value = name_key.to_s, params[name_key]
     # Convert any Time values into UTC/XML schema format (which Solr requires).
     @value = @value.respond_to?(:utc) ? @value.utc.xmlschema : @value.to_s
   end
@@ -28,6 +31,7 @@ class Solr::Field
   def to_xml
     e = Solr::XML::Element.new 'field'
     e.attributes['name'] = @name
+    e.attributes['boost'] = @boost.to_s if @boost
     e.text = @value
     return e
   end

data/lib/solr/importer/array_mapper.rb

@@ -0,0 +1,26 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+class Solr::Importer::ArrayMapper < Solr::Importer::Mapper
+  # TODO document that initializer takes an array of Mappers [mapper1, mapper2, ... mapperN]
+  
+  # TODO: make merge conflict handling configurable.  as is, the last map fields win.
+  def map(orig_data_array)
+    mapped_data = {}
+    orig_data_array.each_with_index do |data,i|
+      mapped_data.merge!(@mapping[i].map(data))
+    end
+    mapped_data
+  end
+end

data/lib/solr/importer/delimited_file_source.rb

@@ -0,0 +1,38 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# For files with the first line containing field names
+# Currently not designed for enormous files, as all lines are
+# read into an array
+class Solr::Importer::DelimitedFileSource
+  include Enumerable
+  
+  def initialize(filename, splitter=/\t/)
+    @filename = filename
+    @splitter = splitter
+  end
+
+  def each
+    lines = IO.readlines(@filename)
+    headers = lines[0].split(@splitter).collect{|h| h.chomp}
+    
+    lines[1..-1].each do |line|
+      data = headers.zip(line.split(@splitter).collect{|s| s.chomp})
+      def data.[](key)
+        self.assoc(key.to_s)[1]
+      end
+      
+      yield(data)
+    end
+  end
+  
+end

data/lib/solr/importer/hpricot_mapper.rb

@@ -0,0 +1,20 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'hpricot'
+
+# For files with the first line containing field names
+class Solr::Importer::HpricotMapper < Solr::Importer::Mapper
+  def field_data(doc, path)
+    doc.search(path.to_s).collect { |e| e.inner_html }
+  end
+end

data/lib/solr/importer/mapper.rb

@@ -0,0 +1,51 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class Solr::Importer::Mapper
+  def initialize(mapping, options={})
+    @mapping = mapping
+    @options = options
+  end
+  
+  def field_data(orig_data, field_name)
+    orig_data[field_name]
+  end
+  
+  def mapped_field_value(orig_data, field_mapping)
+    case field_mapping
+      when String
+        field_mapping
+      when Proc
+        field_mapping.call(orig_data)
+      when Symbol
+        field_data(orig_data, @options[:stringify_symbols] ? field_mapping.to_s : field_mapping)
+      when Enumerable
+        field_mapping.collect {|orig_field_name| mapped_field_value(orig_data, orig_field_name)}.flatten
+      else
+        raise "Unknown mapping for #{field_mapping}"
+    end
+  end
+  
+  def map(orig_data)
+    mapped_data = {}
+    @mapping.each do |solr_name, field_mapping|
+      value = mapped_field_value(orig_data, field_mapping)
+      mapped_data[solr_name] = value if value
+    end
+    
+    mapped_data
+  end
+  
+  
+  
+  
+end

data/lib/solr/importer/solr_source.rb

@@ -0,0 +1,43 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'solr'
+
+class Solr::Importer::SolrSource
+  def initialize(solr_url, query, filter_queries, options={})
+    @connection = Solr::Connection.new(solr_url)
+    @query = query
+    @filter_queries = filter_queries
+
+    @page_size = options[:page_size] || 1000
+    @field_list = options[:field_list] || ["*"]
+  end
+  
+  def each
+    done = false
+    start = 0
+    until done do
+      # request N documents from a starting point
+      request = Solr::Request::Standard.new(:query => @query,
+                                            :rows => @page_size,
+                                            :start => start,
+                                            :field_list => @field_list,
+                                            :filter_queries => @filter_queries)
+      response = @connection.send(request)
+      response.each do |doc|
+        yield doc  # TODO: perhaps convert to HashWithIndifferentAccess.new(doc), so stringify_keys isn't necessary
+      end
+      done = start + @page_size >= response.total_hits
+      start = start + @page_size
+    end
+  end
+end

data/lib/solr/importer/xpath_mapper.rb

@@ -0,0 +1,27 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'xml/libxml'
+
+# For files with the first line containing field names
+class Solr::Importer::XPathMapper < Solr::Importer::Mapper
+  def field_data(doc, xpath)
+    doc.find(xpath.to_s).collect do |node|
+      case node
+        when XML::Attr
+          node.value
+        when XML::Node
+          node.content
+      end
+    end
+  end
+end

data/lib/solr/importer.rb

@@ -0,0 +1,19 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Solr; module Importer; end; end
+require 'solr/importer/mapper'
+require 'solr/importer/array_mapper'
+require 'solr/importer/delimited_file_source'
+require 'solr/importer/hpricot_mapper'
+require 'solr/importer/xpath_mapper'
+require 'solr/importer/solr_source'

data/lib/solr/indexer.rb

@@ -0,0 +1,54 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class Solr::Indexer
+  # deprecated, use Indexer.new(ds,mapping).index instead
+  def self.index(data_source, mapper_or_mapping, options={})
+    indexer = Solr::Indexer.new(data_source, mapper_or_mapping, options)
+    indexer.index
+  end
+  
+  def initialize(data_source, mapper_or_mapping, options={})
+    solr_url = options[:solr_url] || ENV["SOLR_URL"] || "http://localhost:8983/solr"
+    @solr = Solr::Connection.new(solr_url, options) #TODO - these options contain the solr_url and debug keys also, so tidy up what gets passed
+
+    @data_source = data_source
+    @mapper = mapper_or_mapping.is_a?(Hash) ? Solr::Importer::Mapper.new(mapper_or_mapping) : mapper_or_mapping
+
+    @buffer_docs = options[:buffer_docs]
+    @debug = options[:debug]
+  end
+
+  def index
+    buffer = []
+    @data_source.each do |record|
+      document = @mapper.map(record)
+      
+      yield(record, document) if block_given? # TODO check return of block, if not true then don't index
+      
+      buffer << document
+      
+      if !@buffer_docs || buffer.size == @buffer_docs
+        add_docs(buffer)
+        buffer.clear
+      end
+    end
+    add_docs(buffer) if !buffer.empty?
+    
+    @solr.commit unless @debug
+  end
+  
+  def add_docs(documents)
+    @solr.add(documents) unless @debug
+    puts documents.inspect if @debug
+  end
+end

data/lib/solr/request/base.rb

@@ -11,6 +11,9 @@
 # limitations under the License.
 
 class Solr::Request::Base
+  
+  
+  #TODO : Add base support for the debugQuery flag, and such that the response provides debug output easily
 
   # returns either :xml or :ruby depending on what the
   # response type is for a given request
@@ -18,6 +21,10 @@ class Solr::Request::Base
   def response_format
     raise "unknown request type: #{self.class}"
   end
+  
+  def content_type
+    'text/xml; charset=utf-8'
+  end
 
   # returns the solr handler or url fragment that can 
   # respond to this type of request

data/lib/solr/request/commit.rb

@@ -14,8 +14,18 @@ require 'solr/xml'
 
 class Solr::Request::Commit < Solr::Request::Update
 
+  def initialize(options={})
+    @wait_searcher = options[:wait_searcher] || true
+    @wait_flush = options[:wait_flush] || true
+  end
+
+
   def to_s
-    Solr::XML::Element.new('commit').to_s
+    e = Solr::XML::Element.new('commit')
+    e.attributes['waitSearcher'] = @wait_searcher ? 'true' : 'false'
+    e.attributes['waitFlush'] = @wait_flush ? 'true' : 'false'
+    
+    e.to_s
   end
 
 end

data/lib/solr/request/dismax.rb

@@ -16,9 +16,11 @@ class Solr::Request::Dismax < Solr::Request::Standard
                                        :boost_query, :boost_functions])
 
   def initialize(params)
+    @alternate_query = params.delete(:alternate_query)
+    @sort_values = params.delete(:sort)
+    
     super(params)
-    @sort_values = @params[:sort]
-    @params.delete(:sort)
+    
     @query_type = "dismax"
   end
   
@@ -31,6 +33,7 @@ class Solr::Request::Dismax < Solr::Request::Standard
     hash[:ps]  = @params[:phrase_slop]
     hash[:bq]  = @params[:boost_query]
     hash[:bf]  = @params[:boost_functions]
+    hash["q.alt"] = @alternate_query
     # FIXME: 2007-02-13 <coda.hale@gmail.com> --  This code is duplicated in
     # Solr::Request::Standard. It should be refactored into a single location.
     hash[:sort] = @sort_values.collect do |sort|

data/lib/solr/request/index_info.rb

@@ -11,7 +11,9 @@
 # limitations under the License.
 
 class Solr::Request::IndexInfo < Solr::Request::Select
-  def initialize
-    super('indexinfo')
+  
+  def handler
+    'admin/luke'
   end
+  
 end

data/lib/solr/request/select.rb

@@ -31,6 +31,10 @@ class Solr::Request::Select < Solr::Request::Base
   def handler
     'select'
   end
+  
+  def content_type
+    'application/x-www-form-urlencoded; charset=utf-8'
+  end
 
   def to_hash
     return {:qt => query_type, :wt => 'ruby'}

data/lib/solr/request/standard.rb

@@ -71,18 +71,20 @@ class Solr::Request::Standard < Solr::Request::Select
       hash["facet.missing"] = @params[:facets][:missing]
       hash["facet.mincount"] = @params[:facets][:mincount]
       hash["facet.prefix"] = @params[:facets][:prefix]
-      @params[:facets][:fields].each do |f|
-        if f.kind_of? Hash
-          key = f.keys[0]
-          value = f[key]
-          hash["facet.field"] << key
-          hash["f.#{key}.facet.sort"] = (value[:sort] == :count) if value[:sort]
-          hash["f.#{key}.facet.limit"] = value[:limit]
-          hash["f.#{key}.facet.missing"] = value[:missing]
-          hash["f.#{key}.facet.mincount"] = value[:mincount]
-          hash["f.#{key}.facet.prefix"] = value[:prefix]
-        else
-          hash["facet.field"] << f
+      if @params[:facets][:fields]  # facet fields are optional (could be facet.query only)
+        @params[:facets][:fields].each do |f|
+          if f.kind_of? Hash
+            key = f.keys[0]
+            value = f[key]
+            hash["facet.field"] << key
+            hash["f.#{key}.facet.sort"] = (value[:sort] == :count) if value[:sort]
+            hash["f.#{key}.facet.limit"] = value[:limit]
+            hash["f.#{key}.facet.missing"] = value[:missing]
+            hash["f.#{key}.facet.mincount"] = value[:mincount]
+            hash["f.#{key}.facet.prefix"] = value[:prefix]
+          else
+            hash["facet.field"] << f
+          end
         end
       end
     end

data/lib/solr/request/update.rb

@@ -11,6 +11,7 @@
 # limitations under the License.
 
 # a parent class for all requests that go through the solr update handler
+# TODO: Use new xml update handler for better error responses
 class Solr::Request::Update < Solr::Request::Base
   def response_format
     :xml

data/lib/solr/response/commit.rb

@@ -13,20 +13,5 @@
 require 'rexml/xpath'
 
 class Solr::Response::Commit < Solr::Response::Xml
-  attr_reader :ok
-
-  def initialize(xml)
-    super(xml)
-    e = REXML::XPath.first(@doc, './result')
-    if e and e.attributes['status'] == '0'
-      @ok = true
-    else
-      @ok = false
-    end
-  end
-
-  def ok?
-    @ok
-  end
 end
 

data/lib/solr/response/standard.rb

@@ -39,16 +39,15 @@ class Solr::Response::Standard < Solr::Response::Ruby
   def field_facets(field)
     facets = []
     values = @data['facet_counts']['facet_fields'][field]
-    0.upto(values.size / 2 - 1) do |i|
-      n = i * 2
-      facets << FacetValue.new(values[n], values[n+1])
+    Solr::Util.paired_array_each(values) do |key, value|
+      facets << FacetValue.new(key, value)
     end
     
     facets
   end
   
   def highlighted(id, field)
-    @data['highlighting'][id.to_s][field.to_s]
+    @data['highlighting'][id.to_s][field.to_s] rescue nil
   end
   
   # supports enumeration of hits

data/lib/solr/response/xml.rb

@@ -20,11 +20,16 @@ class Solr::Response::Xml < Solr::Response::Base
     super(xml)
     # parse the xml
     @doc = REXML::Document.new(xml)
+
     # look for the result code and string 
-    result = REXML::XPath.first(@doc, './result')
+    # <?xml version="1.0" encoding="UTF-8"?>
+    # <response>
+    # <lst name="responseHeader"><int name="status">0</int><int name="QTime">2</int></lst>
+    # </response>
+    result = REXML::XPath.first(@doc, './response/lst[@name="responseHeader"]/int[@name="status"]')
     if result
-      @status_code =  result.attributes['status']
-      @status_message = result.text
+      @status_code =  result.text
+      @status_message = result.text  # TODO: any need for a message?
     end
   rescue REXML::ParseException => e
     raise Solr::Exception.new("invalid response xml: #{e}")

data/lib/solr/solrtasks.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO: fill out Solr tasks: start, stop, ping, optimize, etc.
+
+require 'rake'
+require 'rake/tasklib'
+
+module Solr
+  namespace :solr do
+    desc "Start Solr"
+    task :start do
+      # TODO: actually start it up!
+      puts "Starting..."
+    end
+  end
+end

data/lib/solr/util.rb

@@ -0,0 +1,34 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class Solr::Util
+  # paired_array_each([key1,value1,key2,value2]) yields twice:
+  #     |key1,value1|  and |key2,value2|
+  def self.paired_array_each(a, &block)
+    0.upto(a.size / 2 - 1) do |i|
+      n = i * 2
+      yield(a[n], a[n+1])
+    end
+  end
+
+  # paired_array_to_hash([key1,value1,key2,value2]) => {key1 => value1, key2, value2}
+  def self.paired_array_to_hash(a)
+    h = {}
+    
+    paired_array_each(a) do |key,value|
+      h[key] = value
+    end
+    
+    h
+  end
+  
+end

data/lib/solr.rb

@@ -15,4 +15,7 @@ require 'solr/exception'
 require 'solr/request'
 require 'solr/connection'
 require 'solr/response'
-require 'solr/xml'
+require 'solr/util'
+require 'solr/xml'
+require 'solr/importer'
+require 'solr/indexer'

data/test/unit/add_document_test.rb

@@ -16,7 +16,7 @@ class AddDocumentTest < SolrMockBaseTestCase
 
   def test_add_document_response
     conn = Solr::Connection.new('http://localhost:9999/solr')
-    set_post_return('<result status="0"></result>')
+    set_post_return('<?xml version="1.0" encoding="UTF-8"?><response><lst name="responseHeader"><int name="status">0</int><int name="QTime">2</int></lst></response>')
     doc = {:id => '123', :text => 'Tlon, Uqbar, Orbis Tertius'}
     response = conn.send(Solr::Request::AddDocument.new(doc))
     assert_equal true, response.ok?
@@ -24,7 +24,7 @@ class AddDocumentTest < SolrMockBaseTestCase
 
   def test_bad_add_document_response
     conn = Solr::Connection.new('http://localhost:9999/solr')
-    set_post_return('<result status="400"></result>')
+    set_post_return('<?xml version="1.0" encoding="UTF-8"?><response><lst name="responseHeader"><int name="status">1</int><int name="QTime">2</int></lst></response>')
     doc = {:id => '123', :text => 'Tlon, Uqbar, Orbis Tertius'}
     response = conn.send(Solr::Request::AddDocument.new(doc))
     assert_equal false, response.ok?
@@ -32,7 +32,7 @@ class AddDocumentTest < SolrMockBaseTestCase
 
   def test_shorthand
     conn = Solr::Connection.new('http://localhost:9999/solr')
-    set_post_return('<result status="0"></result>')
+    set_post_return('<?xml version="1.0" encoding="UTF-8"?><response><lst name="responseHeader"><int name="status">0</int><int name="QTime">2</int></lst></response>')
     doc = {:id => '123', :text => 'Tlon, Uqbar, Orbis Tertius'}
     assert_equal true, conn.add(:id => '123', :text => 'Tlon, Uqbar, Orbis Tetius')
   end

data/test/unit/array_mapper_test.rb

@@ -0,0 +1,37 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'solr'
+require 'test/unit'
+
+include Solr::Importer
+
+class ArrayMapperTest < Test::Unit::TestCase
+  def test_simple
+    mapping1 = {:one => "uno"}
+    mapping2 = {:two => "dos"}
+    
+    mapper = Solr::Importer::ArrayMapper.new([Mapper.new(mapping1),Mapper.new(mapping2)])    
+    mapped_data = mapper.map([{},{}])
+    assert_equal "uno", mapped_data[:one]
+    assert_equal "dos", mapped_data[:two]
+  end
+  
+  def test_field_conflict_goes_to_last
+    mapping1 = {:same => "uno"}
+    mapping2 = {:same => "dos"}
+    
+    mapper = Solr::Importer::ArrayMapper.new([Mapper.new(mapping1),Mapper.new(mapping2)])    
+    mapped_data = mapper.map([{},{}])
+    assert_equal "dos", mapped_data[:same]
+  end
+end

data/test/unit/commit_test.rb

@@ -15,27 +15,27 @@ require 'solr_mock_base'
 class CommitTest < SolrMockBaseTestCase
 
   def test_commit
-    xml = '<result status="0"></result>'
+    xml = '<?xml version="1.0" encoding="UTF-8"?><response><lst name="responseHeader"><int name="status">0</int><int name="QTime">2</int></lst></response>'
     conn = Solr::Connection.new('http://localhost:9999/solr')
     set_post_return(xml)
     response = conn.send(Solr::Request::Commit.new)
     assert_kind_of Solr::Response::Commit, response
-    assert true, response.ok?
+    assert_equal true, response.ok?
 
     # test shorthand
     assert_equal true, conn.commit
   end
 
-  def test_invalid_commit
-    xml = '<foo>bar</foo>'
-    conn = Solr::Connection.new('http://localhost:9999/solr')
-    set_post_return(xml)
-    response = conn.send(Solr::Request::Commit.new)
-    assert_kind_of Solr::Response::Commit, response
-    assert_equal false, response.ok?
-
-    # test shorthand
-    assert_equal false, conn.commit
-   end
+  # def test_invalid_commit
+  #   xml = '<?xml version="1.0" encoding="UTF-8"?><response><lst name="responseHeader"><int name="status">1</int><int name="QTime">2</int></lst></response>'
+  #   conn = Solr::Connection.new('http://localhost:9999/solr')
+  #   set_post_return(xml)
+  #   response = conn.send(Solr::Request::Commit.new)
+  #   assert_kind_of Solr::Response::Commit, response
+  #   assert_equal false, response.ok?
+  # 
+  #   # test shorthand
+  #   assert_equal false, conn.commit
+  #  end
 
 end

data/test/unit/data_mapper_test.rb

@@ -0,0 +1,75 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'solr'
+require 'test/unit'
+
+class DataMapperTest < Test::Unit::TestCase
+
+  def test_static_mapping
+    mapping = {:static => "value",
+               :static_array => ["value1", "value2"]}
+    
+    mapper = Solr::Importer::Mapper.new(mapping)    
+    mapped_data = mapper.map({})
+        
+    assert_equal "value", mapped_data[:static]
+    assert_equal ["value1", "value2"], mapped_data[:static_array]
+  end
+
+  def test_simple_mapping
+    orig_data = {:orig_field => "value",
+                 :multi1 => "val1", :multi2 => "val2"}
+    mapping = {:solr_field => :orig_field,
+               :mapped_array => [:multi1, :multi2], }    
+    
+    mapper = Solr::Importer::Mapper.new(mapping)    
+    mapped_data = mapper.map(orig_data)
+        
+    assert_equal "value", mapped_data[:solr_field]
+    assert_equal ["val1", "val2"], mapped_data[:mapped_array]
+  end
+  
+  def test_proc
+    orig_data = {:orig_field => "value"}
+    mapping = {:solr_field => Proc.new {|record| ">#{record[:orig_field]}<"}}
+    
+    mapper = Solr::Importer::Mapper.new(mapping)    
+    mapped_data = mapper.map(orig_data)
+        
+    assert_equal  ">value<", mapped_data[:solr_field]
+  end
+    
+  def test_overridden_field
+    mapping = {:solr_field => [:orig_field1, :orig_field2]}
+    orig_data = {:orig_field1 => "value1", :orig_field2 => "value2", }
+    
+    mapper = Solr::Importer::Mapper.new(mapping)
+    def mapper.field_data(orig_data, field_name)
+      ["~#{super(orig_data, field_name)}~"]  # array tests that the value is flattened
+    end    
+    mapped_data = mapper.map(orig_data)
+        
+    assert_equal ["~value1~", "~value2~"], mapped_data[:solr_field]
+  end
+  
+  def test_unknown_mapping
+    mapping = {:solr_field => /foo/}  # regexp currently not a valid mapping type
+    
+    mapper = Solr::Importer::Mapper.new(mapping)
+    
+    assert_raise(RuntimeError) do
+      mapped_data = mapper.map({})
+    end
+  end
+
+end

data/test/unit/delete_test.rb

@@ -26,7 +26,7 @@ class DeleteTest <  SolrMockBaseTestCase
 
   def test_delete_response
     conn = Solr::Connection.new 'http://localhost:9999/solr'
-    set_post_return('<result status="0"></result>')
+    set_post_return('<?xml version="1.0" encoding="UTF-8"?><response><lst name="responseHeader"><int name="status">0</int><int name="QTime">2</int></lst></response>')
     response = conn.send(Solr::Request::Delete.new(:id => 123))
     assert_equal true, response.ok? 
   end

data/test/unit/delimited_file_source_test.rb

@@ -0,0 +1,29 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'solr'
+require 'test/unit'
+
+class DelimitedFileSourceTest < Test::Unit::TestCase
+
+  def test_load
+    filename = File.expand_path(File.dirname(__FILE__)) + "/tab_delimited.txt"
+    
+    source = Solr::Importer::DelimitedFileSource.new(filename,/\t/)
+    assert_equal source.to_a.size, 1
+    
+    source.each do |data|
+       assert_equal data[:asin], '0865681740'
+    end
+  end
+
+end

data/test/unit/document_test.rb

@@ -55,5 +55,11 @@ class DocumentTest < Test::Unit::TestCase
     assert_equal 'Lucene in Action', doc[:title]
     assert_equal 'Search', doc[:subject]
   end
+  
+  def test_boost
+    doc = Solr::Document.new :name => "McGrump"
+    doc.boost = 300.28
+    assert_match(/<doc boost=['"]300.28['"]>[\s]+<field name=['"]name['"]>McGrump<\/field>[\s]+<\/doc>/, doc.to_xml.to_s)
+  end
 
 end

data/test/unit/field_test.rb

@@ -39,4 +39,10 @@ class FieldTest < Test::Unit::TestCase
     assert_match(/<field name=["']i18nstring["']>Äêâîôû Öëäïöü<\/field>/m, field.to_xml.to_s)
   end
   
+  def test_boost_values
+    field = Solr::Field.new(:blah => "squee", :boost => 3.0)
+    assert_kind_of Solr::XML::Element, field.to_xml
+    assert_match(/<field name=["']blah["'] boost=["']3.0["']>squee<\/field>/, field.to_xml.to_s)
+  end
+  
 end

data/test/unit/hpricot_mapper_test.rb

@@ -0,0 +1,40 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'solr'
+require 'test/unit'
+require 'hpricot'
+
+class HpricotMapperTest < Test::Unit::TestCase
+  
+  def setup
+    @doc = open(File.expand_path(File.dirname(__FILE__)) + "/hpricot_test_file.xml"){|f| Hpricot.XML(f)}
+  end
+
+  def test_simple_hpricot_path
+    mapping = {:field1 => :'child[@attribute="attribute1"]',
+               :field2 => :'child[@attribute="attribute2"]',
+               :field3 => :'child[@attribute="attribute3"]',
+               :field4 => :'child[@attribute="attribute3"] grandchild',
+               :field5 => :'child'}    
+    
+    mapper = Solr::Importer::HpricotMapper.new(mapping)    
+    mapped_data = mapper.map(@doc)
+        
+    assert_equal ['text1'], mapped_data[:field1]
+    assert_equal ['text2'], mapped_data[:field2]
+    assert_equal ['text3<grandchild>grandchild 3 text</grandchild>'], mapped_data[:field3]
+    assert_equal ['grandchild 3 text'], mapped_data[:field4]
+    assert_equal ['text1', 'text2', 'text3<grandchild>grandchild 3 text</grandchild>'], mapped_data[:field5]
+  end
+
+end

data/test/unit/hpricot_test_file.xml

@@ -0,0 +1,7 @@
+<root>
+  <parent>
+    <child attribute="attribute1">text1</child>
+    <child attribute="attribute2">text2</child>
+    <child attribute="attribute3">text3<grandchild>grandchild 3 text</grandchild></child>
+  </parent>
+</root>

data/test/unit/indexer_test.rb

@@ -0,0 +1,57 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test/unit'
+require 'solr'
+
+class Solr::Indexer
+  attr_reader :added
+  def add_docs(doc)
+    @added ||= []
+    @added << doc
+  end
+end
+
+class IndexerTest < Test::Unit::TestCase
+  def test_mapping_or_mapping
+    mapping = {:field => "foo"}
+    indexer = Solr::Indexer.new([1,2,3], mapping, :debug => true)
+    indexer.index
+    assert_equal 3, indexer.added.size
+    
+    indexer = Solr::Indexer.new([1,2,3,4], Solr::Importer::Mapper.new(mapping), :debug => true)
+    indexer.index
+    assert_equal 4, indexer.added.size
+  end
+
+  def test_batch
+    mapping = {:field => "foo"}
+    indexer = Solr::Indexer.new([1,2,3], mapping, :debug => true, :buffer_docs => 2)
+    indexer.index
+    assert_equal 2, indexer.added.size
+  end
+  
+end
+
+
+# source = DataSource.new
+# 
+# mapping = {
+#   :id => :isbn,
+#   :name => :author,
+#   :source => "BOOKS",
+#   :year => Proc.new {|record| record.date[0,4] },
+# }
+# 
+# Solr::Indexer.index(source, mapper) do |orig_data, solr_document|
+#   solr_document[:timestamp] = Time.now
+# end

data/test/unit/request_test.rb

@@ -22,7 +22,7 @@ class RequestTest < Test::Unit::TestCase
     request = Solr::Request::Commit.new
     assert_equal :xml, request.response_format
     assert_equal 'update', request.handler
-    assert_equal '<commit/>', request.to_s
+    assert_equal '<commit waitSearcher="true" waitFlush="true"/>', request.to_s
   end
   
   def test_add_doc_request

data/test/unit/standard_request_test.rb

@@ -55,6 +55,17 @@ class StandardRequestTest < Test::Unit::TestCase
     assert_match /debugQuery/, request.to_s
   end
   
+  def test_only_facet_query
+    request = Solr::Request::Standard.new(:query => 'query',
+       :facets => {
+         :queries => ["q1", "q2"],
+        }
+    )
+    
+    hash = request.to_hash
+    assert_equal ["q1", "q2"], hash["facet.query"]
+  end
+  
   def test_facet_params_all
     request = Solr::Request::Standard.new(:query => 'query',
        :facets => {

data/test/unit/suite.rb

@@ -10,16 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO dynamically require all *_test files
-
-require 'add_document_test'
-require 'commit_test'
-require 'connection_test'
-require 'delete_test'
-require 'document_test'
-require 'field_test'
-require 'ping_test'
-require 'request_test'
-require 'response_test'
-require 'standard_request_test'
-require 'standard_response_test'
+# dynamically require all tests files
+Dir.glob("*_test.rb").each do | file |
+ require file
+end

data/test/unit/tab_delimited.txt

@@ -0,0 +1,2 @@
+medium	associatedURL	boxHeightInInches	boxLengthInInches	boxWeightInPounds	boxWidthInInches	scannednumber	upc	asin	country	title	fullTitle	series	numberInSeries	edition	aspect	mediacount	genre	price	currentValue	language	netrating	description	owner	publisher	published	rare	purchaseDate	rating	used	signed	hasExperienced	notes	location	paid	condition	notowned	author	illustrator	pages
+book							9780865681743	0865681740	us	Xing Yi Nei Gong: Xing Yi Health Maintenance and Internal Strength Development	Xing Yi Nei Gong: Xing Yi Health Maintenance and Internal Strength Development				Paperback			$21.95	$14.05		4.5	This is the most complete book on the art of xing yi (hsing Yi) available. It includes the complete xing yi history and lineage going back eight generations; manuscripts handed down from famous practitioners Dai Long Bang and Li Neng Ran; 16 health maintenance and power development exercises; qigong (chi kung) exerices; xing yi long spear power training exercises; and more.		Unique Publications	1998-02-10 12:00:00 +0000		2007-02-03 02:22:25 -0500										Dan Miller/ Tim Cartmell		200

data/test/unit/util_test.rb

@@ -0,0 +1,22 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'solr'
+require 'test/unit'
+
+class UtilTest < Test::Unit::TestCase
+
+  def test_paired_array_to_hash
+    assert_equal({:key1 => :value1, :key2 => :value2}, Solr::Util.paired_array_to_hash([:key1, :value1, :key2, :value2]))
+  end
+
+end

data/test/unit/xpath_mapper_test.rb

@@ -0,0 +1,33 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'solr'
+require 'test/unit'
+
+class XPathMapperTest < Test::Unit::TestCase
+  
+  def setup
+    @doc = XML::Document.file(File.expand_path(File.dirname(__FILE__)) + "/xpath_test_file.xml")
+  end
+
+  def test_simple_xpath
+    mapping = {:solr_field1 => :'/root/parent/child',
+               :solr_field2 => :'/root/parent/child/@attribute'}    
+    
+    mapper = Solr::Importer::XPathMapper.new(mapping)    
+    mapped_data = mapper.map(@doc)
+        
+    assert_equal ['text1', 'text2'], mapped_data[:solr_field1]
+    assert_equal ['attribute1', 'attribute2'], mapped_data[:solr_field2]
+  end
+
+end

data/test/unit/xpath_test_file.xml

@@ -0,0 +1,6 @@
+<root>
+  <parent>
+    <child attribute="attribute1">text1</child>
+    <child attribute="attribute2">text2</child>
+  </parent>
+</root>

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.0
+rubygems_version: 0.9.2
 specification_version: 1
 name: solr-ruby
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
-date: 2007-02-15 00:00:00 -05:00
+  version: 0.0.2
+date: 2007-05-14 00:00:00 -04:00
 summary: Ruby library for working with Apache Solr
 require_paths: 
 - lib
-email: solr-user@lucene.apache.org
-homepage: http://wiki.apache.org/solr/Flare
+email: ruby-dev@lucene.apache.org
+homepage: http://wiki.apache.org/solr/solr-ruby
 rubyforge_project: 
 description: 
 autorequire: solr
@@ -35,11 +35,22 @@ files:
 - lib/solr/document.rb
 - lib/solr/exception.rb
 - lib/solr/field.rb
+- lib/solr/importer
+- lib/solr/importer.rb
+- lib/solr/indexer.rb
 - lib/solr/request
 - lib/solr/request.rb
 - lib/solr/response
 - lib/solr/response.rb
+- lib/solr/solrtasks.rb
+- lib/solr/util.rb
 - lib/solr/xml.rb
+- lib/solr/importer/array_mapper.rb
+- lib/solr/importer/delimited_file_source.rb
+- lib/solr/importer/hpricot_mapper.rb
+- lib/solr/importer/mapper.rb
+- lib/solr/importer/solr_source.rb
+- lib/solr/importer/xpath_mapper.rb
 - lib/solr/request/add_document.rb
 - lib/solr/request/base.rb
 - lib/solr/request/commit.rb
@@ -63,12 +74,18 @@ files:
 - lib/solr/response/standard.rb
 - lib/solr/response/xml.rb
 - test/unit/add_document_test.rb
+- test/unit/array_mapper_test.rb
 - test/unit/commit_test.rb
 - test/unit/connection_test.rb
+- test/unit/data_mapper_test.rb
 - test/unit/delete_test.rb
+- test/unit/delimited_file_source_test.rb
 - test/unit/dismax_request_test.rb
 - test/unit/document_test.rb
 - test/unit/field_test.rb
+- test/unit/hpricot_mapper_test.rb
+- test/unit/hpricot_test_file.xml
+- test/unit/indexer_test.rb
 - test/unit/ping_test.rb
 - test/unit/request_test.rb
 - test/unit/response_test.rb
@@ -76,6 +93,10 @@ files:
 - test/unit/standard_request_test.rb
 - test/unit/standard_response_test.rb
 - test/unit/suite.rb
+- test/unit/tab_delimited.txt
+- test/unit/util_test.rb
+- test/unit/xpath_mapper_test.rb
+- test/unit/xpath_test_file.xml
 test_files: []
 
 rdoc_options: []