solrizer 3.0.0.pre6 → 3.0.0.pre7

data/lib/solrizer/default_descriptors.rb

@@ -5,16 +5,21 @@ module Solrizer
     #  _tesim - for strings or text fields
     #  _dtsim - for dates
     #  _isim - for integers
-    # Note that searchable fields are also stored for backwards compatibilitiy 
-    #   TODO:  make a searchable and stored field type  ("searchable_and_displayable" like "search and destroy"?  "displearchable" ?)
-    #     and then make searchable not stored.
+    def self.stored_searchable
+      @stored_searchable ||= Descriptor.new(stored_searchable_field_definition, converter: searchable_converter, requires_type: true)
+    end
+    
+    # The suffix produced depends on the type parameter -- produces suffixes:
+    #  _teim - for strings or text fields
+    #  _dtim - for dates
+    #  _iim - for integers
     def self.searchable
       @searchable ||= Descriptor.new(searchable_field_definition, converter: searchable_converter, requires_type: true)
     end
     
     # Takes fields which are stored as strings, but we want indexed as dates.  (e.g. "November 6th, 2012")
     # produces suffixes:
-    #  _dtsi - for dates
+    #  _dtsim - for dates
     def self.dateable
       @dateable ||= Descriptor.new(:date, :stored, :indexed, :multivalued, converter: dateable_converter)
     end
@@ -54,10 +59,18 @@ module Solrizer
     def self.simple
       @simple ||= Descriptor.new(lambda {|field_type| [field_type, :indexed]})
     end
+    
     protected
 
-    # note that searchable fields are also stored.  Probably should change that at some point
     def self.searchable_field_definition
+      lambda do |type|
+        type = :text_en if [:string, :text].include?(type) # for backwards compatibility with old solr schema
+        vals = [type, :indexed, :multivalued]
+        vals
+      end
+    end
+
+    def self.stored_searchable_field_definition
       lambda do |type|
         type = :text_en if [:string, :text].include?(type) # for backwards compatibility with old solr schema
         vals = [type, :indexed, :stored, :multivalued]

data/lib/solrizer/field_mapper.rb

@@ -113,9 +113,9 @@ module Solrizer
     # @return [String] name of the solr field, based on the params
     def solr_name(field_name, *opts)
       index_type, args = if opts.first.kind_of? Hash
-        [:searchable, opts.first]
+        [:stored_searchable, opts.first]
       elsif opts.empty?
-        [:searchable, {type: :text}]
+        [:stored_searchable, {type: :text}]
       else
         [opts[0], opts[1] || {}]
       end

data/lib/solrizer/version.rb

@@ -1,3 +1,3 @@
 module Solrizer
-  VERSION = "3.0.0.pre6"
+  VERSION = "3.0.0.pre7"
 end

data/lib/solrizer/xml/extractor.rb

@@ -15,14 +15,14 @@ module Solrizer::XML::Extractor
       if value.kind_of?(Array) 
         if value.first.kind_of?(Hash)
           # This deals with the way xml-simple handles nodes with attributes
-          solr_doc.merge!({mapper.solr_name(name, :searchable, :type=>:text).to_sym => "#{value.first["content"]}"})
+          solr_doc.merge!({mapper.solr_name(name, :stored_searchable, :type=>:text).to_sym => "#{value.first["content"]}"})
         elsif value.length > 1
-          solr_doc.merge!({mapper.solr_name(name, :searchable, :type=>:text).to_sym => value})
+          solr_doc.merge!({mapper.solr_name(name, :stored_searchable, :type=>:text).to_sym => value})
         else
-          solr_doc.merge!({mapper.solr_name(name, :searchable, :type=>:text).to_sym => "#{value.first}"})
+          solr_doc.merge!({mapper.solr_name(name, :stored_searchable, :type=>:text).to_sym => "#{value.first}"})
         end
       else
-        solr_doc.merge!({mapper.solr_name(name, :searchable, :type=>:text).to_sym => "#{value}"})
+        solr_doc.merge!({mapper.solr_name(name, :stored_searchable, :type=>:text).to_sym => "#{value}"})
       end
     end
 

data/lib/solrizer/xml/terminology_based_solrizer.rb

@@ -36,6 +36,7 @@ module Solrizer::XML::TerminologyBasedSolrizer
       
       nodeset.each do |n|
         doc.solrize_node(n, term_pointer, term, solr_doc, field_mapper)
+# FIXME:  there should be no dependencies on OM in Solrizer
         unless term.kind_of? OM::XML::NamedTermProxy
           term.children.each_pair do |child_term_name, child_term|
             doc.solrize_term(child_term, solr_doc, field_mapper, opts={:parents=>parents+[{term.name=>nodeset.index(n)}]})
@@ -57,10 +58,12 @@ module Solrizer::XML::TerminologyBasedSolrizer
     def solrize_node(node_value, doc, term_pointer, term, solr_doc = Hash.new, field_mapper = nil, opts = {})
       return solr_doc unless term.index_as && !term.index_as.empty?
       
+# FIXME:  there should be no dependencies on OM in Solrizer
       generic_field_name_base = OM::XML::Terminology.term_generic_name(*term_pointer)
       create_and_insert_terms(generic_field_name_base, node_value, term.index_as, solr_doc)
       
       if term_pointer.length > 1
+# FIXME:  there should be no dependencies on OM in Solrizer
         hierarchical_field_name_base = OM::XML::Terminology.term_hierarchical_name(*term_pointer)
         create_and_insert_terms(hierarchical_field_name_base, node_value, term.index_as, solr_doc)
       end

data/lib/solrizer.rb

@@ -35,7 +35,7 @@ module Solrizer
   # @returns [Hash] doc the document that was provided with the new field inserted
   def self.insert_field(doc, name, value, *indexer_args)
     # adding defaults indexer 
-    indexer_args = [:searchable] if indexer_args.empty?
+    indexer_args = [:stored_searchable] if indexer_args.empty?
     default_field_mapper.solr_names_and_values(name, value, indexer_args).each do |k, v|
       doc[k] ||= []
       doc[k] += v
@@ -50,7 +50,7 @@ module Solrizer
   # @returns [Hash] doc the document that was provided with the new field (replacing any field with the same name) 
   def self.set_field(doc, name, value, *indexer_args)
     # adding defaults indexer 
-    indexer_args = [:searchable] if indexer_args.empty?
+    indexer_args = [:stored_searchable] if indexer_args.empty?
     doc.merge! default_field_mapper.solr_names_and_values(name, value, indexer_args)
     doc
   end

data/spec/fixtures/mods_article.rb

@@ -13,16 +13,16 @@ module Samples
         }
         t.french_title(:ref=>[:title_info,:main_title], :attributes=>{"xml:lang"=>"fre"})
         
-        t.language(:index_as=>[:facetable, :searchable],:path=>{:attribute=>"lang"})
+        t.language(:index_as=>[:facetable, :stored_searchable],:path=>{:attribute=>"lang"})
       } 
       t.language{
         t.lang_code(:index_as=>[:facetable], :path=>"languageTerm", :attributes=>{:type=>"code"})
       }
-      t.abstract(:index_as=>[:searchable])
+      t.abstract(:index_as=>[:stored_searchable])
       t.subject {
         t.topic(:index_as=>[:facetable])
       }      
-      t.topic_tag(:proxy=>[:subject, :topic], :index_as=>[:searchable])    
+      t.topic_tag(:proxy=>[:subject, :topic], :index_as=>[:stored_searchable])    
       # t.topic_tag(:index_as=>[:facetable],:path=>"subject", :default_content_path=>"topic")
       # This is a mods:name.  The underscore is purely to avoid namespace conflicts.
       t.name_ {
@@ -35,7 +35,7 @@ module Samples
         t.role(:ref=>[:role])
         t.description(:index_as=>[:facetable])
         t.date(:path=>"namePart", :attributes=>{:type=>"date"})
-        t.last_name(:path=>"namePart", :attributes=>{:type=>"family"}, :index_as=>[:searchable])
+        t.last_name(:path=>"namePart", :attributes=>{:type=>"family"}, :index_as=>[:stored_searchable])
         t.first_name(:path=>"namePart", :attributes=>{:type=>"given"}, :label=>"first name")
         t.terms_of_address(:path=>"namePart", :attributes=>{:type=>"termsOfAddress"})
         t.computing_id
@@ -47,14 +47,14 @@ module Samples
       t.organization(:ref=>:name, :attributes=>{:type=>"corporate"}, :index_as=>[:facetable])
       t.conference(:ref=>:name, :attributes=>{:type=>"conference"}, :index_as=>[:facetable])
       t.role {
-        t.text(:path=>"roleTerm",:attributes=>{:type=>"text"}, :index_as=>[:searchable])
+        t.text(:path=>"roleTerm",:attributes=>{:type=>"text"}, :index_as=>[:stored_searchable])
         t.code(:path=>"roleTerm",:attributes=>{:type=>"code"})
       }
       t.journal(:path=>'relatedItem', :attributes=>{:type=>"host"}) {
         t.title_info(:index_as=>[:facetable],:ref=>[:title_info])
         t.origin_info(:path=>"originInfo") {
           t.publisher
-          t.date_issued(:path=>"dateIssued", :type => :date, :index_as => [:searchable])
+          t.date_issued(:path=>"dateIssued", :type => :date, :index_as => [:stored_searchable])
           t.issuance(:index_as=>[:facetable])
         }
         t.issn(:path=>"identifier", :attributes=>{:type=>"issn"})
@@ -68,7 +68,7 @@ module Samples
           }
           t.start_page(:proxy=>[:pages, :start])
           t.end_page(:proxy=>[:pages, :end])
-          t.publication_date(:path=>"date", :type => :date, :index_as => [:searchable])
+          t.publication_date(:path=>"date", :type => :date, :index_as => [:stored_searchable])
         }
       }
       t.note

data/spec/units/common_spec.rb

@@ -13,11 +13,12 @@ describe Solrizer::Common do
   it "should handle many field types" do
     solr_doc = {}
     Foo.create_and_insert_terms('my_name', 'value', [:displayable, :searchable, :sortable], solr_doc)
-    solr_doc.should == {'my_name_ssm' => ['value'], 'my_name_si' => ['value'], 'my_name_tesim' => ['value']}
+    solr_doc.should == {'my_name_ssm' => ['value'], 'my_name_si' => ['value'], 'my_name_teim' => ['value']}
   end
+  
   it "should handle dates that are searchable" do
     solr_doc = {}
-    Foo.create_and_insert_terms('my_name', Date.parse('2013-01-10'), [:searchable], solr_doc)
+    Foo.create_and_insert_terms('my_name', Date.parse('2013-01-10'), [:stored_searchable], solr_doc)
     solr_doc.should == {'my_name_dtsim' => ['2013-01-10T00:00:00Z']}
   end
 

data/spec/units/field_mapper_spec.rb

@@ -11,14 +11,14 @@ describe Solrizer::FieldMapper do
         @unstemmed_searchable ||= UnstemmedDescriptor.new()
       end
 
-      # Produces a _s suffix (overrides _tim)
-      def self.searchable
-        @searchable ||= SearchableDescriptor.new()
+      # Produces a _s suffix (overrides _tesim)
+      def self.stored_searchable
+        @searchable ||= StoredSearchableDescriptor.new()
       end
 
-      # Produces a _s suffix (overrides _tim)
-      def self.another_searchable
-        @another_searchable ||= SearchableDescriptor.new()
+      # Produces a _s suffix (overrides _tesim)
+      def self.another_stored_searchable
+        @another_searchable ||= StoredSearchableDescriptor.new()
       end
 
       def self.edible
@@ -39,7 +39,7 @@ describe Solrizer::FieldMapper do
         end
       end
 
-      class SearchableDescriptor < Solrizer::Descriptor
+      class StoredSearchableDescriptor < Solrizer::Descriptor
         def name_and_converter(field_name, args)
           [field_name.to_s + '_s']
         end
@@ -156,7 +156,7 @@ describe Solrizer::FieldMapper do
       @mapper.solr_name('bar', :laughable, type: :string).should == 'bar_haha'
     end
 
-    it "should default the index_type to :searchable" do
+    it "should default the index_type to :stored_searchable" do
       @mapper.solr_name('foo').should == 'foo_s'
     end
 
@@ -191,7 +191,7 @@ describe Solrizer::FieldMapper do
   
   describe '.solr_names_and_values' do
     it "should map values based on passed descriptors" do
-      @mapper.solr_names_and_values('foo', 'bar', [:searchable, :laughable, :edible]).should == {
+      @mapper.solr_names_and_values('foo', 'bar', [:stored_searchable, :laughable, :edible]).should == {
         'foo_s'    => ['bar'],
         'foo_food' => ['bar'],
         'foo_haha' => ["Knock knock. Who's there? Bar. Bar who?"]
@@ -199,7 +199,7 @@ describe Solrizer::FieldMapper do
     end
     
     it "should apply mappings based on data type" do
-      @mapper.solr_names_and_values('foo', 7, [:searchable, :laughable]).should == {
+      @mapper.solr_names_and_values('foo', 7, [:stored_searchable, :laughable]).should == {
         'foo_s'     => ['7'],
         'foo_ihaha' => ["How many foos does it take to screw in a light bulb? 7."]
       }
@@ -212,19 +212,19 @@ describe Solrizer::FieldMapper do
     end
     
     it "should generate multiple mappings when two return the _same_ solr name but _different_ values" do
-      @mapper.solr_names_and_values('roll', 'rock', [:unstemmed_searchable, :searchable]).should == {
+      @mapper.solr_names_and_values('roll', 'rock', [:unstemmed_searchable, :stored_searchable]).should == {
         'roll_s' => ["rock o'clock", 'rock']
       }
     end
     
     it "should not generate multiple mappings when two return the _same_ solr name and the _same_ value" do
-      @mapper.solr_names_and_values('roll', 'rock', [:another_searchable, :searchable]).should == {
+      @mapper.solr_names_and_values('roll', 'rock', [:another_stored_searchable, :stored_searchable]).should == {
         'roll_s' => ['rock'],
       }
     end
 
     it "should return an empty hash when value is nil" do
-      @mapper.solr_names_and_values('roll', nil, [:another_searchable, :searchable]).should == { }
+      @mapper.solr_names_and_values('roll', nil, [:another_stored_searchable, :stored_searchable]).should == { }
     end
   end
 
@@ -237,7 +237,7 @@ describe Solrizer::FieldMapper do
       @mapper.id_field.should == 'id'
     end
 
-    it "should default the index_type to :searchable" do
+    it "should default the index_type to :stored_searchable" do
       @mapper.solr_name('foo', :type=>:string).should == 'foo_tesim'
     end
     
@@ -258,12 +258,15 @@ describe Solrizer::FieldMapper do
       @mapper.solr_names_and_values('foo', '', [:dateable]).should == { 'foo_dtsim' => [] }
     end
 
-    it "should support searchable, displayable, facetable, sortable, unstemmed" do
-      @mapper.solr_names_and_values('foo', 'bar', [:searchable, :displayable, :facetable, :sortable, :unstemmed_searchable]).should == {
-        "foo_tesim" => ["bar"], #searchable
+    it "should support searchable, stored_searchable, displayable, facetable, sortable, stored_sortable, unstemmed" do
+      descriptors = [:searchable, :stored_searchable, :displayable, :facetable, :sortable, :stored_sortable, :unstemmed_searchable]
+      @mapper.solr_names_and_values('foo', 'bar', descriptors).should == {
+        "foo_teim" => ["bar"], #searchable
+        "foo_tesim" => ["bar"], #stored_searchable
         "foo_ssm" => ["bar"], #displayable
         "foo_sim" => ["bar"], #facetable
         "foo_si" => ["bar"], #sortable
+        "foo_ssi" => ["bar"], #stored_sortable
         "foo_tim" => ["bar"] #unstemmed_searchable
       }
     end
@@ -272,7 +275,6 @@ describe Solrizer::FieldMapper do
       time = Time.iso8601("2012-11-06T15:16:17Z")
       @mapper.solr_names_and_values('foo', time, :stored_sortable).should == {"foo_dtsi" => ["2012-11-06T15:16:17Z"]} 
       @mapper.solr_names_and_values('foo', 'bar', :stored_sortable).should == {"foo_ssi" => ["bar"]} 
-
     end
   end
 end

data/spec/units/solrizer_spec.rb

@@ -5,7 +5,7 @@ describe Solrizer do
   describe ".insert_field" do
     describe "on an empty document" do
       let(:doc) { Hash.new }
-      it "should insert a field with the default (searchable) indexer" do
+      it "should insert a field with the default (stored_searchable) indexer" do
         Solrizer.insert_field(doc, 'foo', 'A name')
         doc.should == {'foo_tesim' => ['A name']}
       end

data/spec/units/xml_extractor_spec.rb

@@ -20,7 +20,6 @@ describe Solrizer::XML::Extractor do
       result[:format_tesim].should include("application/jp2000")
       result[:title_tesim].should == "This is a Sample Title"
       result[:publisher_tesim].should == "Sample Unversity"
-
     end
   end
   

data/spec/units/xml_terminology_based_solrizer_spec.rb

@@ -1,6 +1,7 @@
 require 'spec_helper'
 require 'fixtures/mods_article'
 
+# TODO:  there should be no dependencies on OM in Solrizer
 describe Solrizer::XML::TerminologyBasedSolrizer do
   
   before(:all) do
@@ -34,9 +35,9 @@ describe Solrizer::XML::TerminologyBasedSolrizer do
     end
   
     it "should use Solr mappings to generate field names" do
-
       solr_doc =  @mods_article.to_solr
       solr_doc["abstract"].should be_nil
+      # NOTE:  OM's old default expected stored and indexed;  this is a change.
       solr_doc["abstract_tesim"].should == ["ABSTRACT"]
       solr_doc["title_info_1_language_tesim"].should == ["finnish"]
       solr_doc["person_1_role_0_text_tesim"].should == ["teacher"]
@@ -44,13 +45,10 @@ describe Solrizer::XML::TerminologyBasedSolrizer do
       solr_doc["person_1_role_0_code_tesim"].should be_nil 
       solr_doc["person_last_name_tesim"].sort.should == ["FAMILY NAME", "Gautama"]
       solr_doc["topic_tag_tesim"].sort.should == ["CONTROLLED TERM", "TOPIC 1", "TOPIC 2"]
-      
       # These are a holdover from an old verison of OM
       solr_doc['journal_0_issue_0_publication_date_dtsim'].should == ["2007-02-01T00:00:00Z"]
-
-      
     end
-    
+
   end
 
   describe ".solrize_term" do
@@ -69,7 +67,7 @@ describe Solrizer::XML::TerminologyBasedSolrizer do
       @mods_article.solrize_term(term, fake_solr_doc)
       
       expected_names = ["DR.", "FAMILY NAME", "GIVEN NAMES"]
-      %w(_tesim _sim).each do |suffix|
+      %w(_teim _sim).each do |suffix|
         actual_names = fake_solr_doc["name_0_namePart#{suffix}"].sort
         actual_names.should == expected_names
       end
@@ -95,7 +93,7 @@ describe Solrizer::XML::TerminologyBasedSolrizer do
       term.children[:namePart].index_as = []
 
       @mods_article.solrize_term(term, fake_solr_doc)
-      fake_solr_doc["name_0_namePart_tesim"].should be_nil
+      fake_solr_doc["name_0_namePart_teim"].should be_nil
     end
 
     it "should index terms where index_as is searchable" do
@@ -105,7 +103,7 @@ describe Solrizer::XML::TerminologyBasedSolrizer do
 
       @mods_article.solrize_term(term, fake_solr_doc)
       
-      fake_solr_doc["name_0_namePart_tesim"].sort.should == ["DR.", "FAMILY NAME", "GIVEN NAMES"]
+      fake_solr_doc["name_0_namePart_teim"].sort.should == ["DR.", "FAMILY NAME", "GIVEN NAMES"]
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solrizer
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre6
+  version: 3.0.0.pre7
   prerelease: 6
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-30 00:00:00.000000000 Z
+date: 2013-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -207,8 +207,6 @@ files:
 - SOLRIZING_OM_DOCUMENTS.textile
 - bin/solrizer
 - bin/solrizerd
-- config/solr_mappings.yml
-- config/solr_mappings_af_0.1.yml
 - lib/solrizer.rb
 - lib/solrizer/common.rb
 - lib/solrizer/default_descriptors.rb
@@ -251,9 +249,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 4544422725210300531
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/config/solr_mappings.yml

@@ -1,16 +0,0 @@
-id: id
-searchable: 
-  default: _t
-  date: _dt
-  string: _t
-  text: _t
-  symbol: _s
-  integer: _i
-  long: _l
-  boolean: _b
-  float: _f
-  double: _d
-displayable: _display
-facetable: _facet
-sortable: _sort
-unstemmed_searchable: _unstem_search

data/config/solr_mappings_af_0.1.yml

@@ -1,18 +0,0 @@
-id: id
-default: searchable
-searchable:
-  date: _date
-  string: _field
-  text: _field
-  symbol: _field
-  integer: _field
-  long: _field
-  boolean: _field
-  float: _field
-  double: _field
-displayable: _display
-facetable: _facet
-sortable: _sort
-unstemmed_searchable: _unstem_search
-
-