lucid_works 0.7.10 → 0.7.15

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lucid_works (0.7.7)
+    lucid_works (0.7.14)
       activemodel (>= 3)
       activesupport (>= 3)
       json

data/config/locales/en.yml

@@ -42,7 +42,7 @@ en:
             index_time_stopwords: Exclude stop words from index
             query_parser: Query parser
             query_time_stopwords: Include stop words in searches
-            query_time_synonyms: Use synomyms
+            query_time_synonyms: Use synonyms
             search_server_list: Search server list
             show_similar: Show "find similar" links
             spellcheck: Spell-check
@@ -54,6 +54,9 @@ en:
             unsupervised_feedback_emphasis: Unsupervised feedback emphasis
             update_server_list: Update server list
         datasource:
+          subtype:
+            web:
+              url: Custom legend for url
           aliases: Site Aliases Mappings
           auth: Authentication credentials
           authorization: Authorization
@@ -83,7 +86,6 @@ en:
           log_extra_detail: Log extra detail
           max_bytes: Skip files larger than (bytes)
           my_site_base_url: MySite URL
-          no_duplicates: Suppress duplicates
           password: Password
           proxy_host: Proxy host
           proxy_password: Proxy password
@@ -121,7 +123,7 @@ en:
           include_in_results: Include in results
           index_for_autocomplete: Index for autocomplete
           index_for_spellcheck: Index for spell checking
-          index_term_freq_and_pos: Index term frequencies and positions
+          indexing_options: Index term frequencies and positions
           indexed: Indexed
           multi_valued: Multi-valued
           name: Name
@@ -194,7 +196,7 @@ en:
         field:
           one: Field
           other: Fields
-          index_term_freq_and_pos:
+          indexing_options:
             document_only: none
             document_termfreq: term frequencies
             document_termfreq_termpos: term frequencies and positions
@@ -212,14 +214,50 @@ en:
       collection:
         name: ! 'Collection names may use the characters: A-Z, a-z, 0-9, dash and
           underscore'
+
       datasource:
         crawl_depth: Leave blank or set to -1 to crawl with no depth limit.
         exclude_paths: A list of regular expressions, one per line, e.g. .*\.pdf will
           ignore filenames ending in .pdf.
-        include_paths: A list of regular expressions, one per line, e.g. http://example\.com/.*
-          will match everything within the site only.
+        include_paths: A list of regular expressions, one per line.
         max_bytes: Default = 10 MiB. Set to -1 to indicate no file size limit.
-        url: e.g. http://cnn.com. Please include protocol (http/https).
+
+      # Custom datasource hints
+      external:
+      file:
+        datasource:
+          path: Full pathname to folder
+      ftp:
+      hdfs:
+        datasource:
+          url: "Please include protocol: hdfs://<hostname>"
+      jdbc:
+        datasource:
+          delta_sql_query: "$ in this query will be replaced by the last successful import time."
+          driver: "Database drivers may be uploaded on the Indexing -> JDBC Drivers page.
+            Drivers are collection specific.
+            JDBC4 drivers will appear in the selector automatically.
+            To use a JDBC3 driver, select \"Other\" and enter the Java class name of the driver."
+          url: "e.g. jdbc:mysql://hostname/database_name"
+      kfs:
+      lucidworkslogs:
+      s3:
+        datasource:
+          url: "Please include protocol: s3://<hostname>/path/to/filesystem"
+      s3n:
+        datasource:
+          url: "Please include protocol: s3n://<hostname>/path/to/filesystem"
+      sharepoint:
+      smb:
+        datasource:
+          url: "Format: smb://<hostname or ip address>/<path to folder>/"
+      solrxml:
+      web:
+        datasource:
+          include_paths: A list of regular expressions, one per line, e.g. http://example\.com/.*
+            will match everything within the site only.
+          url: e.g. http://cnn.com. Please include protocol (http/https).
+
       field:
         copy_fields: A list of fields separated by spaces.
       synonym:

data/lib/lucid_works/activity/history.rb

@@ -4,12 +4,8 @@ module LucidWorks
       belongs_to :activity
       self.collection_name = 'history' # i.e. not the plural 'histories'
 
-      def activity_finished
-        Time.iso8601 activityFinished
-      end
-
-      def activity_started
-        Time.iso8601 activityStarted
+      schema do
+        attributes :activity_started, :activity_finished, :type => :iso8601
       end
 
       def duration

data/lib/lucid_works/base.rb

@@ -27,6 +27,7 @@ module LucidWorks
     # wandering through that code.
 
     include ActiveModel::Validations
+    include ActiveModel::Validations::Callbacks
     include ActiveModel::Conversion
     extend  ActiveModel::Naming
     extend  ActiveModel::Translation
@@ -294,8 +295,8 @@ module LucidWorks
     end
 
     def save
+      return false unless valid?
       _run_save_callbacks do
-        return false unless valid?
         ActiveSupport::Notifications.instrument("lucid_works.request") do |payload|
           method, uri = persisted? ? [:put, member_url] : [:post, collection_url]
           data = encode

data/lib/lucid_works/collection.rb

@@ -20,7 +20,7 @@ module LucidWorks
       "filterer.class" => "com.lucid.security.WindowsACLQueryFilterer",
       "provider.class" => "com.lucid.security.ad.ADACLTagProvider",
     }
-    MAGIC_ACL_ONLY_FILTER_SETTING = {'should.clause' => '*:* -data_source_type:smb'}
+    MAGIC_ACL_ONLY_FILTER_SETTING = {'should_clause' => '*:* -data_source_type:smb'}
     
     validates_presence_of :name
 

data/lib/lucid_works/crawler.rb

@@ -10,7 +10,7 @@ module LucidWorks
 
     def datasource_types=(array_of_hashes)
       @datasource_types ||= array_of_hashes.map do |dt_attrs|
-        DatasourceType.new(dt_attrs)
+        DatasourceType.new(self, dt_attrs)
       end
     end
 

data/lib/lucid_works/datasource.rb

@@ -59,7 +59,6 @@ module LucidWorks
       attribute :log_extra_detail, :boolean
       attribute :fail_unsupported_file_types, :boolean
       attribute :warn_unknown_mime_types, :boolean
-      attribute :no_duplicates, :boolean
       # sharepoint
       attribute :sharepoint_url
       attribute :connector_type

data/lib/lucid_works/datasource_type.rb

@@ -2,9 +2,16 @@ module LucidWorks
 
   class DatasourceType
 
-    attr_reader :category, :type, :props
+    DONT_INITIALIZE_PROPERTIES = [
+      "collection",  # not something we can provide a default for
+      "name",        # not something we can provide a default for
+      "---"          # Separator UI hint
+    ]
 
-    def initialize(attributes = {})
+    attr_reader :crawler, :category, :type, :props
+
+    def initialize(crawler, attributes = {})
+      @crawler  = crawler
       @category = attributes['category']
       @type     = attributes['type']
       @props    = attributes['props']
@@ -19,5 +26,19 @@ module LucidWorks
     def property(name)
       properties.detect { |p| p.name == name }
     end
+
+    def default_attributes
+      attrs = {}.with_indifferent_access
+      properties.inject(attrs) do |hash, property|
+        unless DONT_INITIALIZE_PROPERTIES.include?(property.name) || property.read_only?
+          hash[property.name] = property.default_value
+        end
+        hash
+      end
+      attrs.merge!(
+        :crawler => @crawler.name,
+        :type => @type
+      )
+    end
   end
 end

data/lib/lucid_works/field.rb

@@ -9,6 +9,12 @@ module LucidWorks
       'document_termfreq_termpos'
     ]
 
+    BOOST_VALUES = [
+      'none',
+      'moderate',
+      'high'
+    ]
+
     schema do
       attribute :name, :string, :primary_key => true, :omit_during_update => true
       attribute :editable, :boolean, :omit_during_update => true
@@ -24,16 +30,9 @@ module LucidWorks
       attribute :default_boost, :integer
       attributes :field_type, :short_field_boost, :term_vectors, :default_value, :type => :string
       attribute :copy_fields, :list
-
-      attribute :index_term_freq_and_pos, :custom, :values => INDEXING_OPTIONS
+      attribute :indexing_options, :custom, :values => INDEXING_OPTIONS
     end
 
-    BOOST_VALUES = [
-      'none',
-      'moderate',
-      'high'
-    ]
-
     validates_presence_of :name
     validates_each :name, :unless => :persisted?, :allow_blank => true do |model, attr, value|
       model.errors.add(attr, 'must be unique') if model.collection.fields.any? {|f| f.name == value }
@@ -53,11 +52,34 @@ module LucidWorks
     validates_each :use_in_find_similar do |model, attr, value|
       model.errors.add(attr, 'a field must be indexed for it to be used for find-similar') if value == true && !model.indexed?
     end
+    validates_each :indexing_options do |model, attr, value|
+      if model.persisted?
+        if value
+          if model.indexed?
+            model.errors.add(attr, "must be one of #{INDEXING_OPTIONS.join(', ')}") unless INDEXING_OPTIONS.include?(value.to_s)
+          end
+        else
+          # If the model is already persisted, but the user has not manually set indexing_options,
+          # just go with the omit_tf/omit_positions we have now.
+        end
+      else # new model
+        if model.indexed?
+          # If this is a new model, and the user set 'indexed' they MUST also set indexing_options
+          model.errors.add(attr, "must be one of #{INDEXING_OPTIONS.join(', ')}") unless INDEXING_OPTIONS.include?(value.to_s)
+        end
+      end
+    end
+
+    before_save :convert_indexing_options_to_term_freq_and_pos
+
+    def initialize(options)
+      super(options.reverse_merge(:omit_tf => false, :short_field_boost => 'high'))
+    end
 
     def dynamically_generated?
       !dynamic_base.blank?
     end
-    
+
     def t_field_type
       self.class.t_field_type(self.field_type)
     end
@@ -65,20 +87,39 @@ module LucidWorks
     def self.t_field_type(type)
       I18n.translate(type, :scope => 'activemodel.models.lucid_works.collection.field.field_type')
     end
-    
-    def initialize(options)
-      super(options.reverse_merge(:omit_tf => false, :short_field_boost => 'high'))
+
+    # Indexing_options is a fake attribute that wraps the omit_tf and omit_positions combinations.
+    # Translations are:
+    #
+    #   INDEXED?     indexing_options                    omit_tf         omit_positions
+    #
+    #   false           -                                 true              true
+    #   true            :document_only                    true              true
+    #   true            :document_termfreq                false             true
+    #   true            :documents_termfreq_termpos       false             false
+    #
+    def indexing_options
+      @attributes[:indexing_options] ||=
+        if !indexed? || omit_tf?
+          :document_only
+        else
+          omit_positions? ? :document_termfreq : :document_termfreq_termpos
+        end
     end
 
-    # Meta attribute that wraps the omit_tf and omit_positions combinations
-    def index_term_freq_and_pos
-      return :document_only if !indexed? || omit_tf?
-      return omit_positions? ? :document_termfreq : :document_termfreq_termpos
+    # We have to remember the caller's choice of indexing_options, as we can't determine the correct
+    # omit_tf/omit_positions settins until we also know whether 'indexed' is set or not.
+    # So we keep it as an attribute for now, and remove it in a before_save.
+    def indexing_options=(new_value)
+      @attributes[:indexing_options] = new_value
     end
 
-    def index_term_freq_and_pos=(indexing_options)
-      if indexed?
-        case indexing_options.to_sym
+    # Convert indexing_options to the correct values for omit_tf and omit_positions.
+    # Delete indexing_options from attributes so it is not saved.
+    def convert_indexing_options_to_term_freq_and_pos
+      i_opt = @attributes.delete(:indexing_options)
+      if indexed? && i_opt
+        case i_opt.to_sym
         when :document_only
           self.omit_tf = true
           self.omit_positions = true
@@ -89,12 +130,13 @@ module LucidWorks
           self.omit_tf = false
           self.omit_positions = false
         else
-          raise "Unknown indexing option: '#{indexing_options}'. Allowed values are: #{INDEXING_OPTIONS.join(', ')}"
+          raise "Unknown indexing option: '#{i_opt}'. Allowed values are: #{INDEXING_OPTIONS.join(', ')}"
         end
       else # !indexed?
         self.omit_tf = true
         self.omit_positions = true
       end
+      true
     end
 
     def update_attributes(attrs)

data/lib/lucid_works/gem_version.rb

@@ -1,3 +1,3 @@
 module LucidWorks
-  VERSION = "0.7.10"
+  VERSION = "0.7.15"
 end

data/spec/lib/lucid_works/field_spec.rb

@@ -4,7 +4,7 @@ describe LucidWorks::Field do
   before :all do
     @server = connect_to_live_server
     @server.reset_collections!
-    @collection = @server.collections.first
+    @collection = @server.collection('collection1')
   end
 
   it "should use the ORM" do
@@ -80,7 +80,7 @@ describe LucidWorks::Field do
             field.should_not be_valid
             field.errors[:short_field_boost].should == ['cannot be set to "none" for a non-indexed field']
           end
-          LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => true, :short_field_boost => true).should be_valid
+          LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => true, :short_field_boost => true, :indexing_options => :document_only).should be_valid
         end
       end
 
@@ -90,7 +90,7 @@ describe LucidWorks::Field do
             field.should_not be_valid
             field.errors[:include_in_results].should == ["a field must be stored to be included in results"]
           end
-          LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :stored => true, :include_in_results => true).should be_valid
+          LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :stored => true, :include_in_results => true, :indexing_options => :document_only).should be_valid
         end
       end
 
@@ -110,7 +110,7 @@ describe LucidWorks::Field do
             field.should_not be_valid
             field.errors[:facet].should == ["a field must be indexed to be facetable"]
           end
-          LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => true, :facet => true).should be_valid
+          LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => true, :facet => true, :indexing_options => :document_only).should be_valid
         end
       end
 
@@ -120,7 +120,7 @@ describe LucidWorks::Field do
             field.should_not be_valid
             field.errors[:use_in_find_similar].should == ["a field must be indexed for it to be used for find-similar"]
           end
-          LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => true, :use_in_find_similar => true).should be_valid
+          LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => true, :use_in_find_similar => true, :indexing_options => :document_only).should be_valid
         end
       end
     end
@@ -133,101 +133,141 @@ describe LucidWorks::Field do
     end
   end
 
-  describe "#index_term_freq_and_pos" do
+  describe "indexing options" do
+    INDEXING_OPTIONS_TRUTH_TABLE = [
+      { :indexed => true,  :indexing_options => :document_termfreq_termpos, :omit_tf => false, :omit_positions => false },
+      { :indexed => true,  :indexing_options => :document_termfreq,         :omit_tf => false, :omit_positions => true },
+      { :indexed => true,  :indexing_options => :document_only,             :omit_tf => true,  :omit_positions => true },
+      { :indexed => false, :indexing_options => :document_termfreq_termpos, :omit_tf => true, :omit_positions => true },
+      { :indexed => false, :indexing_options => :document_termfreq_termpos, :omit_tf => true, :omit_positions => true },
+      { :indexed => false, :indexing_options => :document_termfreq_termpos, :omit_tf => true, :omit_positions => true },
+    ]
 
-    context "when indexed is false" do
-      before do
-        @field = LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => false)
-      end
+    context "when neither the setter nor getter have been accessed" do
+      context "for an object with indexed = false" do
+        before do
+          @field = LucidWorks::Field.find('url', :parent => @collection)
+          @field.indexed.should be_false
+        end
 
-      [
-        { :omit_tf => false, :omit_positions => false },
-        { :omit_tf => false, :omit_positions => true  },
-        { :omit_tf => true,  :omit_positions => false },
-        { :omit_tf => true,  :omit_positions => true  }
-      ].each do |settings|
-        it "should reuturn :document if omit_tf=#{settings[:omit_tf]} and omit_positions=#{settings[:omit_positions]}" do
-          settings.each { |k,v| @field.send("#{k}=", v) }
-          @field.index_term_freq_and_pos.should == :document_only
+        it "should ignore indexing_options and validate okay" do
+          @field.should be_valid
         end
-      end
-    end
 
-    context "when indexed is true" do
-      before do
-        @field = LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => true)
+        it "should save omit_tf and omit_positions as true" do
+          RestClient.stub(:send) do |method, url, payload, options|
+            method.should == :put
+            hash = JSON.parse(payload)
+            hash['omit_positions'].should be_true
+            hash['omit_tf'].should be_true
+          end
+
+          @field.save
+        end
       end
 
-      [
-        { :omit_tf => false, :omit_positions => false, :expected_result => :document_termfreq_termpos },
-        { :omit_tf => false, :omit_positions => true , :expected_result => :document_termfreq },
-        { :omit_tf => true,  :omit_positions => false, :expected_result => :document_only },
-        { :omit_tf => true,  :omit_positions => true , :expected_result => :document_only }
-      ].each do |data|
-        it "should return #{data[:expected_result]} if omit_tf=#{data[:omit_tf]} and omit_positions=#{data[:omit_positions]}" do
-          @field.omit_tf = data[:omit_tf]
-          @field.omit_positions = data[:omit_positions]
-          @field.index_term_freq_and_pos.should == data[:expected_result]
+      context "for an object with indexed = true" do
+        before do
+          @field = LucidWorks::Field.find('body', :parent => @collection)
+        end
+
+        it "should validate correctly" do
+          @field.should be_valid
         end
       end
     end
-  end
 
-  describe "#index_term_freq_and_pos=" do
+    describe "getter" do
+      context "when indexed is false" do
+        before do
+          @field = LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => false)
+        end
 
-    INDEX_TERM_FREQ_AND_POS_TRUTH_TABLE = [
-      { :omit_tf => false, :omit_positions => false, :index_term_freq_and_pos => :document_termfreq_termpos },
-      { :omit_tf => false, :omit_positions => true , :index_term_freq_and_pos => :document_termfreq },
-      { :omit_tf => true,  :omit_positions => true , :index_term_freq_and_pos => :document_only }
-    ]
+        describe "it should always return :document_termfreq_termpos" do
+          INDEXING_OPTIONS_TRUTH_TABLE.select{ |x| x[:indexed] == 'false'}.each do |settings|
+            it "should return :document_termfreq_termpos if omit_tf=#{settings[:omit_tf]} and omit_positions=#{settings[:omit_positions]}" do
+              settings.each { |k,v| @field.send("#{k}=", v) }
+              @field.indexing_options.should == :document_termfreq_termpos
+            end
+          end
 
-    context "when indexed is true" do
-      before do
-        @field = LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => true)
+        end
       end
 
-      INDEX_TERM_FREQ_AND_POS_TRUTH_TABLE.each do |data|
-        it "should set omit_tf=#{data[:omit_tf]} and omit_positions=#{data[:omit_positions]} when given #{data[:index_term_freq_and_pos]}" do
-          #                       Set to opposite so we can see change
-          @field.omit_tf        = !data[:omit_tf]
-          @field.omit_positions = !data[:omit_positions]
-
-          @field.index_term_freq_and_pos = data[:index_term_freq_and_pos]
-          
-          @field.omit_positions.should == data[:omit_positions]
-          @field.omit_tf.should        == data[:omit_tf]
+      context "when indexed is true" do
+        before do
+          @field = LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => true)
         end
-      end
 
-      context "given anything else" do
-        it "should raise an error" do
-          lambda {
-            @field.index_term_freq_and_pos = :documents_and_bananas
-          }.should raise_error
+        INDEXING_OPTIONS_TRUTH_TABLE.select { |x| x[:indexed] == true }.each do |data|
+          it "should return #{data[:indexing_options]} if omit_tf=#{data[:omit_tf]} and omit_positions=#{data[:omit_positions]}" do
+            @field.omit_tf = data[:omit_tf]
+            @field.omit_positions = data[:omit_positions]
+            @field.indexing_options.should == data[:indexing_options]
+          end
         end
       end
+
     end
 
-    context "when indexed is false" do
+    describe "setter" do
       before do
-        @field = LucidWorks::Field.new(:parent => @collection, :name => 'my_field', :indexed => false)
+        @field = LucidWorks::Field.new(:parent => @collection, :name => 'my_field')
       end
 
-      INDEX_TERM_FREQ_AND_POS_TRUTH_TABLE.each do |data|
-        it "should set omit_tf=true and omit_positions=true when given #{data[:index_term_freq_and_pos]}" do
-          #                       Set to opposite so we can see change
-          @field.omit_tf        = false
-          @field.omit_positions = false
+      INDEXING_OPTIONS_TRUTH_TABLE.each do |data|
+
+        context "when indexed is set to #{data[:indexed]} BEFORE indexing options is set to #{data[:indexing_options]}" do
+          before do
+            @field.indexed = data[:indexed]
+            @field.indexing_options = data[:indexing_options]
+          end
 
-          @field.index_term_freq_and_pos = data[:index_term_freq_and_pos]
+          it "should set omit_tf=#{data[:omit_tf]} and omit_positions=#{data[:omit_positions]} before save" do
+            RestClient.stub(:send) do |method, url, payload, options|
+              method.should == :post
+              hash = JSON.parse(payload)
+              hash['omit_tf'].should == data[:omit_tf]
+              hash['omit_positions'].should == data[:omit_positions]
+            end
+            @field.save
+          end
 
-          @field.omit_positions.should == true
-          @field.omit_tf.should        == true
+          it "should not pass indexing_options on to the rest api" do
+            RestClient.stub(:send) do |method, url, payload, options|
+              method.should == :post
+              hash = JSON.parse(payload)
+              hash.keys.should_not include('indexing_options')
+            end
+            @field.save
+          end
         end
-      end
 
-      it "should set omit_tf=true and omit_positions=true" do
+        context "when indexed is set to #{data[:indexed]} AFTER indexing options is set to #{data[:indexing_options]}" do
+          before do
+            @field.indexing_options = data[:indexing_options]
+            @field.indexed = data[:indexed]
+          end
+
+          it "should set omit_tf=#{data[:omit_tf]} and omit_positions=#{data[:omit_positions]} before save" do
+            RestClient.stub(:send) do |method, url, payload, options|
+              method.should == :post
+              hash = JSON.parse(payload)
+              hash['omit_tf'].should == data[:omit_tf]
+              hash['omit_positions'].should == data[:omit_positions]
+            end
+            @field.save
+          end
 
+          it "should not pass indexing_options on to the rest api" do
+            RestClient.stub(:send) do |method, url, payload, options|
+              method.should == :post
+              hash = JSON.parse(payload)
+              hash.keys.should_not include('indexing_options')
+            end
+            @field.save
+          end
+        end
       end
     end
   end

metadata

@@ -2,7 +2,7 @@
 name: lucid_works
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.7.10
+  version: 0.7.15
 platform: ruby
 authors: 
 - Sam Pierson
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-02 00:00:00 -04:00
+date: 2011-09-12 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency