lucid_works 0.6.29 → 0.7.1

data/lib/lucid_works/collection/click.rb

@@ -0,0 +1,17 @@
+require 'lucid_works/field'
+
+module LucidWorks
+  class Collection
+    class Click < Base
+      self.singleton = true
+      belongs_to :collection
+
+      schema do
+        attribute :type, :string, :values => %w{ q c }
+        attributes :req, :q, :doc, :type => :string
+        attributes :qt, :ct, :hits, :pos, :type => :integer
+        attribute :buffering, :boolean
+      end
+    end
+  end
+end

data/lib/lucid_works/collection/settings.rb

@@ -16,7 +16,6 @@ module LucidWorks
         # Indexing Settings
         attribute  :unknown_type_handling, :string, :values => LucidWorks::Field::TYPES
         attribute  :de_duplication, :string, :values => DEDUP_OPTIONS
-        attribute  :index_time_stopwords, :boolean
 
         # Querying Settings
         attribute :unsupervised_feedback_emphasis, :string, :values => FEEDBACK_EMPHASIS

data/lib/lucid_works/collection.rb

@@ -3,8 +3,8 @@ module LucidWorks
   class Collection < Base
 
     belongs_to :server
-    has_many :datasources, :fields, :activities
-    has_one  :info, :settings
+    has_many :datasources, :fields, :fieldtypes, :activities, :roles
+    has_one  :info, :settings, :click
     has_one  :index, :has_content => false
 
     schema do
@@ -16,7 +16,6 @@ module LucidWorks
     LOGS_COLLECTION_NAME = 'LucidWorksLogs'
 
     validates_presence_of :name
-    validates_exclusion_of :name, :in => [LOGS_COLLECTION_NAME]
 
     def destroyable?
       true
@@ -113,5 +112,25 @@ module LucidWorks
     def autocomplete_activity
       prime_activities.detect{|act| act.type == 'autocomplete'}
     end
+
+    # URL of Solr's build-in admin page
+    def admin_url
+      "#{server.host}/solr/#{name}/admin/"
+    end
+
+    # We have to handle the jdbcdrivers model in a custom way, as a GET on .../jdbcdrivers returns a list of strings,
+    # but then you can actually do a DELETE on .../jdbcdrivers/name.
+    # For the DELETE reason we do use a LucidWorks::Base model, but we have to create them by hand.
+    def jdbcdrivers
+      driverlist = JSON.parse RestClient.get(self.uri + '/jdbcdrivers')
+      driverlist.map do |drivername|
+        Jdbcdriver.new(:name => drivername, :collection => self)
+      end
+    end
+
+    def jdbcdriver_classes
+      JSON.parse RestClient.get(uri + "/jdbcdrivers/classes")
+    end
+
   end
 end

data/lib/lucid_works/crawler.rb

@@ -3,6 +3,19 @@ module LucidWorks
   class Crawler < Base
     schema do
       primary_key :name
+      attribute :datasource_types, :custom
+    end
+
+    attr_reader :datasource_types
+
+    def datasource_types=(array_of_hashes)
+      @datasource_types ||= array_of_hashes.map do |dt_attrs|
+        DatasourceType.new(dt_attrs)
+      end
+    end
+
+    def datasource_type(type)
+      datasource_types.detect { |t| t.type == type }
     end
   end
 end

data/lib/lucid_works/datasource/history.rb

@@ -4,17 +4,13 @@ module LucidWorks
     class History < Base
       belongs_to :datasource
       self.collection_name = 'history' # i.e. not the plural 'histories'
-
-      def doc_count
-        numUpdated + numNew + numUnchanged
+      schema do
+        attributes :crawl_started, :crawl_stopped, :type => :iso8601
+        attributes :num_updated, :num_new, :num_unchanged, :type => :integer
       end
 
-      def crawl_stopped
-        Time.iso8601 crawlStopped
-      end
-
-      def crawl_started
-        Time.iso8601 crawlStarted
+      def doc_count
+        num_updated + num_new + num_unchanged
       end
 
       def duration

data/lib/lucid_works/datasource/status.rb

@@ -10,9 +10,10 @@ module LucidWorks
       CRAWLSTATES =            STOPPED_STATES + [ 'RUNNING' ] + POST_PROCESSING_STATES
 
       schema do
-        attribute :crawlState, :string, :values => CRAWLSTATES
-        attributes :crawlStarted, :crawlStopped, :jobId
-        attributes :numUnchanged, :numUpdated, :numNew, :numFailed, :numDeleted, :numTotal, :type => :integer
+        attribute :crawl_state, :string, :values => CRAWLSTATES
+        attribute :jobId
+        attributes :crawl_started, :crawl_stopped, :type => :iso8601
+        attributes :num_unchanged, :num_updated, :num_new, :num_failed, :num_deleted, :num_total, :type => :integer
       end
 
       # Create predicate methods for all the crawl states
@@ -20,25 +21,21 @@ module LucidWorks
         method_name = state.downcase + "?"
         class_eval <<-EOF
           def #{method_name}
-            self.crawlState == "#{state}"
+            self.crawl_state == "#{state}"
           end
         EOF
       end
 
       def stopped?
-        STOPPED_STATES.include?(crawlState)
+        STOPPED_STATES.include?(crawl_state)
       end
 
       def post_processing?
-        POST_PROCESSING_STATES.include?(crawlState)
+        POST_PROCESSING_STATES.include?(crawl_state)
       end
 
       def doc_count
-        numUpdated + numNew + numUnchanged
-      end
-
-      def crawl_started
-        Time.iso8601 crawlStarted
+        num_updated + num_new + num_unchanged
       end
 
       def elapsed_time

data/lib/lucid_works/datasource.rb

@@ -2,46 +2,96 @@ module LucidWorks
   
   class Datasource < Base
     belongs_to :collection
-    has_many   :histories, :class_name => :history, :supports_all => true
-    has_one :status, :supports_all => true
+    has_many   :histories, :class_name => :history, :retrieveable_en_masse => true
+    has_one :status, :retrieveable_en_masse => true
     has_one :schedule, :crawldata
     has_one :index, :job, :has_content => false
 
+    TYPES = %w{ external file ftp hdfs kfs lucidworkslogs web s3 s3n smb solrxml jdbc sharepoint }
+    SYSTEM_TYPES = %w{ lucidworkslogs }
+    BOUNDS = %w{ tree none }
+    CRAWLERS = {
+      # Later we may change these to be arrays if we decide to support more than one choice
+      # e.g. :web => ['lucid.aperture', 'nutch'], :file => ['lucid.aperture', 'lucid.fs']
+      :file           => 'lucid.aperture',
+      :lucidworkslogs => 'lucid.logs',
+      :external       => 'lucid.external',
+      :web            => 'lucid.aperture',
+      :solrxml        => 'lucid.solrxml',
+      :jdbc           => 'lucid.jdbc',
+      :sharepoint     => 'lucid.gcm',
+      :ftp            => 'lucid.fs',
+      :hdfs           => 'lucid.fs',
+      :kfs            => 'lucid.fs',
+      :smb            => 'lucid.fs',
+      :s3n            => 'lucid.fs',
+      :s3             => 'lucid.fs'
+    }.with_indifferent_access
+
     schema do
       # common
-      attributes :name, :type, :crawler
+      attributes :name, :crawler
+      attribute :type, :string, :values => TYPES
       attributes :crawl_depth, :max_docs,     :type => :integer
-      attributes :max_bytes,   :commitWithin, :type => :integer, :omit_when_blank => true
+      attributes :max_bytes,   :commit_within, :type => :integer, :omit_when_blank => true
       attribute :commit_within_min, :custom
+      attribute :commit_on_finish, :boolean
       attribute :include_paths
       attribute :exclude_paths
-      attribute :mapping # Hash
-      attribute :bounds
+      attribute :mapping, :string, :omit_when_blank => true # Hash
+      attribute :bounds, :string, :values => BOUNDS
       # web
       attributes :url, :category
       attribute :collect_links, :boolean
+      attribute :auth, :string, :omit_when_blank => true # Hash
+      attributes :proxy_host, :string, :omit_when_blank => true
+      attribute :proxy_port, :string, :omit_when_blank => true
+      attribute :proxy_username, :string, :omit_when_blank => true
+      attribute :proxy_password, :string, :omit_when_blank => true
+      attribute :ignore_robots, :boolean
       # file
       attribute :path
       attribute :follow_links, :boolean
+      attribute :parsing, :boolean
+      attribute :indexing, :boolean
+      attribute :caching, :boolean
+      attribute :verify_access, :boolean
+      attribute :log_extra_detail, :boolean
+      attribute :fail_unsupported_file_types, :boolean
+      attribute :warn_unknown_mime_types, :boolean
+      attribute :no_duplicates, :boolean
       # sharepoint
-      attribute :sharepointUrl
-      attribute :connectorType
+      attribute :sharepoint_url
+      attribute :connector_type
       attribute :authorization
       attribute :username
       attribute :password
       attribute :domain
-      attribute :mySiteBaseURL, :string, :nil_when_blank => true
-      attribute :includedURls
-      attribute :excludedURls
+      attribute :my_site_base_url, :string, :nil_when_blank => true
+      attribute :included_urls
+      attribute :excluded_urls
       attribute :kdcserver
-      attribute :useSPSearchVisibility, :boolean
+      attribute :use_sp_search_visibility, :boolean
       attribute :aliases
       # external
       attribute :source
       attribute :source_type
       # lucidworkslogs
-      attribute :deleteAfter, :integer, :omit_when_blank => true
+      attribute :delete_after, :integer, :omit_when_blank => true
       attribute :delete_after_days, :custom
+      # smb
+      attribute :windows_domain
+      attributes :parsing, :indexing, :caching, :type => :boolean
+      # jdbc
+      attribute :driver
+      attribute :primary_key
+      attribute :sql_select_statement
+      attribute :delta_sql_query
+      attribute :nested_queries # list
+      # solrxml
+      attribute :file
+      attribute :include_datasource_metadata, :boolean
+      attribute :generate_unique_key, :boolean
     end
 
     validates_presence_of :type, :crawler, :name
@@ -49,33 +99,18 @@ module LucidWorks
     validates_numericality_of :max_bytes, :allow_blank => true
     validates_presence_of :url, :if => lambda { |d| d.type == 'web' }
 
-    TYPES = %w{ external file lucidworkslogs web solrxml jdbc sharepoint }
-    BOUNDS = %w{ tree none }
-    CRAWLERS = {
-      # Later we may change these to be arrays if we decide to support more than one choice
-      # e.g. :web => ['lucid.aperture', 'nutch']
-      :file           => 'lucid.aperture',
-      :lucidworkslogs => 'lucid.logs',
-      :external       => 'lucid.external',
-      :web            => 'lucid.aperture',
-      :solrxml        => 'lucid.solrxml',
-      :jdbc           => 'lucid.jdbc',
-      :sharepoint     => 'lucid.gcm'
-    }.with_indifferent_access
-
-
     # Fake attributes to ease UI implementation
     def commit_within_min
-      commitWithin.blank? ? nil : commitWithin / 1.second.milliseconds / 1.minute.seconds
+      commit_within.blank? ? nil : commit_within / 1.second.milliseconds / 1.minute.seconds
     end
     def commit_within_min=(mins)
-      self.commitWithin = mins.blank? ? nil : mins.to_i.minutes.milliseconds
+      self.commit_within = mins.blank? ? nil : mins.to_i.minutes.milliseconds
     end
     def delete_after_days
-      deleteAfter.blank? ? nil : deleteAfter / 1.second.milliseconds / 1.day.seconds
+      delete_after.blank? ? nil : deleteAfter / 1.second.milliseconds / 1.day.seconds
     end
     def delete_after_days=(days)
-      self.deleteAfter = days.blank? ? nil : days.to_i.days.milliseconds
+      self.delete_after = days.blank? ? nil : days.to_i.days.milliseconds
     end
 
     def document_count

data/lib/lucid_works/datasource_property.rb

@@ -0,0 +1,18 @@
+module LucidWorks
+
+  class DatasourceProperty
+
+    attr_reader :description, :name, :allowed_values, :type, :default_value, :required, :read_only
+    alias :read_only? :read_only
+
+    def initialize(attributes = {})
+      @description    = attributes['description']
+      @name           = attributes['name']
+      @allowed_values = attributes['allowedValues']
+      @type           = attributes['type']
+      @default_value  = attributes['defaultValues']
+      @required       = attributes['required']
+      @read_only      = attributes['read_only']
+    end
+  end
+end

data/lib/lucid_works/datasource_type.rb

@@ -0,0 +1,23 @@
+module LucidWorks
+
+  class DatasourceType
+
+    attr_reader :category, :type, :props
+
+    def initialize(attributes = {})
+      @category = attributes['category']
+      @type     = attributes['type']
+      @props    = attributes['props']
+    end
+
+    def properties
+      @properties ||= @props.map do |prop|
+        DatasourceProperty.new(prop)
+      end
+    end
+
+    def property(name)
+      properties.detect { |p| p.name == name }
+    end
+  end
+end

data/lib/lucid_works/exceptions.rb

@@ -3,4 +3,5 @@ module LucidWorks
   end
 
   class ResourceNotFound < Exception ; end
+  class RecordInvalid < Exception ; end
 end

data/lib/lucid_works/field.rb

@@ -3,12 +3,19 @@ module LucidWorks
   class Field < Base
     belongs_to :collection
 
+    INDEXING_OPTIONS = [
+      'document_only',
+      'document_termfreq',
+      'document_termfreq_termpos'
+    ]
+
     schema do
       attribute :name, :string, :primary_key => true, :omit_during_update => true
       attribute :editable, :boolean, :omit_during_update => true
+      attribute :dynamic_base, :string, :omit_during_update => true
       attributes :indexed, :stored, :facet, :include_in_results,
                  :search_by_default, :highlight, :multi_valued,
-                 :term_vectors, :omit_tf,
+                 :term_vectors, :omit_tf, :omit_positions,
                  :use_for_deduplication, :synonym_expansion,
                  :index_for_spellcheck, :index_for_autocomplete,
                  :query_time_stopword_handling,
@@ -16,6 +23,8 @@ module LucidWorks
                  :type => :boolean
       attribute :default_boost, :integer
       attributes :field_type, :short_field_boost, :term_vectors, :default_value, :copy_fields, :type => :string
+
+      attribute :index_term_freq_and_pos, :custom, :values => INDEXING_OPTIONS
     end
 
     TYPES = [
@@ -82,6 +91,10 @@ module LucidWorks
     validates_each :use_in_find_similar do |model, attr, value|
       model.errors.add(attr, 'a field must be indexed for it to be used for find-similar') if value == true && !model.indexed?
     end
+
+    def dynamically_generated?
+      !dynamic_base.blank?
+    end
     
     def t_field_type
       self.class.t_field_type(self.field_type)
@@ -94,7 +107,34 @@ module LucidWorks
     def initialize(options)
       super(options.reverse_merge(:omit_tf => false, :short_field_boost => 'high'))
     end
-    
+
+    # Meta attribute that wraps the omit_tf and omit_positions combinations
+    def index_term_freq_and_pos
+      return :document_only if !indexed? || omit_tf?
+      return omit_positions? ? :document_termfreq : :document_termfreq_termpos
+    end
+
+    def index_term_freq_and_pos=(indexing_options)
+      if indexed?
+        case indexing_options.to_sym
+        when :document_only
+          self.omit_tf = true
+          self.omit_positions = true
+        when :document_termfreq
+          self.omit_tf = false
+          self.omit_positions = true
+        when :document_termfreq_termpos
+          self.omit_tf = false
+          self.omit_positions = false
+        else
+          raise "Unknown indexing option: '#{indexing_options}'. Allowed values are: #{INDEXING_OPTIONS.join(', ')}"
+        end
+      else # !indexed?
+        self.omit_tf = true
+        self.omit_positions = true
+      end
+    end
+
     def update_attributes(attrs)
       attrs = attrs.with_indifferent_access
       if [false, '0'].include?(attrs[:stored]) 
@@ -105,6 +145,7 @@ module LucidWorks
         attrs[:facet] ||= false
         attrs[:synonym_expansion] ||= false
         attrs[:omit_tf] ||= false 
+        attrs[:omit_positions] ||= false
         attrs[:short_field_boost] ||= 'high'
         attrs[:search_by_default] ||= false
         attrs[:use_in_find_similar] ||= false

data/lib/lucid_works/fieldtype.rb

@@ -0,0 +1,28 @@
+module LucidWorks
+
+  class Fieldtype < Base
+    belongs_to :collection
+
+    ANALYZER_TYPES = %w{ default index query }
+
+    schema do
+      dynamic_attributes true
+      attribute :name, :string, :primary_key => true, :omit_during_update => true
+      attribute :class, :string
+      attribute :analyzers
+    end
+
+    validates_presence_of :name, :_class
+
+    # Provide support for adding more attributes during update.
+    def update_attributes(attrs)
+      attrs.keys.each do |attr|
+        unless self.class.schema.has_attribute?(attr)
+          self.class.schema.add_attribute(attr, :string) unless self.class.schema.has_attribute?(attr)
+        end
+      end
+
+      super
+    end
+  end
+end