lucid_works 0.3.9 → 0.4.9

data/README.rdoc

@@ -41,8 +41,7 @@ This single statement (note the periods) will connect to a LucidWorks server run
     create_datasource(:name => 'cnn',
                       :crawler => 'lucid.aperture', :type => 'web',
                       :url => 'http://cnn.com', :crawl_depth => '1').
-    build_schedule(:start_time => 0, :period => 0, :type => 'index', :active => true).
-    save
+    start_crawl
 
 Now, how does it work:
 
@@ -56,10 +55,13 @@ The LucidWorks object model looks something like this:
           |              |              +- Schedule
           |              |              +- Index
           |              |              +- Crawldata
+          |              |              +- Job
           |              +- Field
           |              +- Index
           |              +- Info
           |              +- Settings
+          |              +- Activity -+- Status
+          |                           +- History
           |
           +- Logs -+- Index -+- Summary
           |        +- Query -+- Summary
@@ -150,6 +152,14 @@ Collection has_many :datasources.  Datasources are modeled using the LucidWorks:
   
 Note that the latter does not start a crawl of the datasource.
 
+To start a datasource crawling:
+
+  datasource.start_crawl!
+
+To stop a datasource crawl:
+
+  datasource.stop_crawl!
+  
 To delete all the data crawled from a data-source:
 
   datasource.empty!
@@ -247,7 +257,27 @@ Then:
 
   whatnot.thing         -> A Thing
 
-=== Rationale
+=== Schema
+
+A class may have a schema defined as follows:
+
+  class ThingWithSchema < LucidWorks::Base
+    schema do
+      attribute :string1,  :string
+      attribute :bool1,    :boolean
+      attribute :integer1, :integer
+      attributes :string2, :string3, :string4
+      attributes :bool2, :bool3, :type => :boolean
+      attributes :int2,  :int3,  :type => :integer
+      attribute :string_with_values, :values => ['one', 'two']
+      attribute :dontsendme, :omit_during_update => true
+    end
+  end
+
+Classes with a schema may have validations applied to its attributes.
+The default attribute type is :string.
+
+== Rationale
 
 Originally this library started out as a set of ActiveResource classes.  This required a lot of hacking of ActiveResource as ActiveResource makes a lot of assumptions about the way a REST API should work - it's basically just designed to talk to Rails applications - and many REST APIs, including this one, don't conform to those rules.  Among the changes required to ActiveResource were:
 

data/config/locales/en.yml

@@ -4,6 +4,12 @@ en:
     models:
       lucid_works:
         collection:
+          activity:
+            type:
+              optimize: Optimize
+              spelling: Spelling
+              click: Click
+              autocomplete: Autocomplete
           one: Collection
           other: Collections
           settings:
@@ -14,21 +20,23 @@ en:
               overwrite: Overwrite
               tag: Tag
         datasource:
-          one: Data-source
-          other: Data-sources
+          one: Data source
+          other: Data sources
           status:
-            crawl_state:
-              aborted: Aborted
-              aborting: Aborting
-              exception: Exception
-              finished: Finished
-              idle: Idle
-              running: Running
-              stopped: Stopped
-              stopping: Stoppong
+            crawlState:
+              ABORTED: Aborted
+              ABORTING: Aborting
+              EXCEPTION: Exception
+              FINISHED: Finished
+              IDLE: Idle
+              RUNNING: Running
+              STOPPED: Stopped
+              STOPPING: Stopping
           type:
+            external: External
             file: Local Filesystem
             jdbc: Database
+            lwelogs: LucidWorks Solr Logs
             sharepoint: Sharepoint
             solrxml: Solr XML
             web: Web Site
@@ -66,7 +74,7 @@ en:
             de_duplication: De-duplication
             display_facets: Display facets
             elevations: elevations
-            index_time_stopwords: Don't index stop words
+            index_time_stopwords: Exclude stop words from index
             query_parser: Query parser
             query_time_stopwords: Include stop words in searches
             query_time_synonyms: Use synomyms
@@ -104,3 +112,4 @@ en:
             numDeleted: Deleted docs
             numUnchanged: Unchanged docs
             numFailed: Failed docs
+            numTotal: Total docs

data/lib/lucid_works.rb

@@ -5,6 +5,7 @@ end
 
 require 'active_model'
 require 'active_support/core_ext/module/attr_accessor_with_default'
+require 'active_support/core_ext/module/aliasing'
 require 'active_support/core_ext/hash/indifferent_access'
 require 'active_support/inflector'
 begin
@@ -13,8 +14,12 @@ rescue LoadError
 end
 require 'restclient'
 require 'json'
+require 'rsolr'
+require 'nokogiri'
 
+require 'lucid_works/utils'
 require 'lucid_works/patch_restclient'
+require 'lucid_works/patch_time'
 
 require 'lucid_works/exceptions'
 require 'lucid_works/associations'
@@ -23,6 +28,10 @@ require 'lucid_works/base'
 require 'lucid_works/schema'
 
 require 'lucid_works/collection'
+require 'lucid_works/collection/activity'
+require 'lucid_works/collection/activity/status'
+require 'lucid_works/collection/activity/history'
+#require 'lucid_works/collection/activity/schedule'
 require 'lucid_works/collection/info'
 require 'lucid_works/collection/index'
 require 'lucid_works/collection/settings'
@@ -33,6 +42,7 @@ require 'lucid_works/datasource/status'
 require 'lucid_works/datasource/history'
 require 'lucid_works/datasource/schedule'
 require 'lucid_works/datasource/crawldata'
+require 'lucid_works/datasource/job'
 require 'lucid_works/field'
 require 'lucid_works/logs'
 require 'lucid_works/logs/query'

data/lib/lucid_works/associations.rb

@@ -89,26 +89,28 @@ module LucidWorks
       def define_has_one(resource, options={})
         resource_class_name = (options[:class_name] || resource).to_s.camelize
 
+        class_eval <<-EOF, __FILE__, __LINE__ + 1
+          def #{resource}                                                  # def child
+            @#{resource} || #{resource}!                                   #   @child || child!
+          end                                                              # end
+        EOF
+
         if options[:has_content] == false
-          class_eval <<-EOF1, __FILE__, __LINE__ + 1
-            def #{resource}                                                # def resource
-              #{resource_class_name}.new(:parent => self)                  #   Child.new(options.merge :parent => self)
+          class_eval <<-EOF, __FILE__, __LINE__ + 1
+            def #{resource}!                                               # def child!
+              @#{resource} = #{resource_class_name}.new(:parent => self)   #   @child = Child.new(options.merge :parent => self)
             end                                                            # end
-          EOF1
+          EOF
         else
-          class_eval <<-EOF2, __FILE__, __LINE__ + 1
-            def #{resource}                                                # def resource
-              @#{resource} || #{resource}!                                 #   @resource || resource!
-            end                                                            # end
-
-            def #{resource}!                                               # def resource!
-              @#{resource} = #{resource_class_name}.find(:parent => self)  #   @resource = Resource.find(:parent => self)
+          class_eval <<-EOF, __FILE__, __LINE__ + 1
+            def #{resource}!                                               # def child!
+              @#{resource} = #{resource_class_name}.find(:parent => self)  #   @child = Child.find(:parent => self)
             end                                                            # end
 
             def build_#{resource}(options = {})
               #{resource_class_name}.new(options.merge :parent => self)
             end
-          EOF2
+          EOF
         end
       end
 

data/lib/lucid_works/base.rb

@@ -32,6 +32,7 @@ module LucidWorks
     extend  ActiveModel::Translation
     extend  ActiveModel::Callbacks
     include Associations
+    include Utils::BoolConverter
 
     attr_accessor :parent           # :nodoc:
     attr_writer   :id               # :nodoc:
@@ -44,10 +45,8 @@ module LucidWorks
     class << self
       include ActionView::Helpers::NumberHelper rescue nil
 
-      attr_accessor_with_default :primary_key, :id
       attr_accessor :collection_name # :nodoc:
       attr_accessor_with_default :singleton, false
-      attr_accessor_with_default :has_schema, false
 
       # The attributes for a model are ascertained in on of two ways.
       # Without a schema, the attributes list is automatically generated when the the object is retrieved from the server.
@@ -69,7 +68,6 @@ module LucidWorks
         if block_given?
           @schema.instance_eval(&block)
           @schema.create_accessors_for_attributes(self)
-          self.has_schema = true
         end
         @schema
       end
@@ -182,8 +180,8 @@ module LucidWorks
         find(:all, options).last
       end
 
-      # Convert the attribute value to a string.  If a schema has been defined for the modeland a type has
-      # been defined for the attribute, it will have formatting applied as follows:
+      # Convert the attribute value to a string.  If a schema has been defined for the model
+      # and a type has been defined for the attribute, it will have formatting applied as follows:
       #
       # - <tt>boolean</tt> will be converted to 'yes' or 'no'
       # - <tt>integer</tt> will be passed to number_with_delimter
@@ -238,7 +236,7 @@ module LucidWorks
         else
           parent = options.delete(:parent)
         end
-        raise ArgumentError.new("parent is a required option") unless parent
+        raise ArgumentError.new("parent is a required option (options were #{options.inspect}") unless parent
         unless parent.is_a?(Base) || parent.is_a?(Server)
           raise ArgumentError.new("parent must be a LucidWorks::Server or LucidWorks::Base")
         end
@@ -297,11 +295,11 @@ module LucidWorks
     end
 
     def id # :nodoc:
-      @attributes[self.class.primary_key]
+      @attributes[self.class.schema.primary_key]
     end
 
     def id=(value) # :nodoc:
-      @attributes[self.class.primary_key] = value
+      @attributes[self.class.schema.primary_key] = value
     end
 
     def persisted?
@@ -348,7 +346,9 @@ module LucidWorks
     end
 
     def encode # :nodoc:
-      @attributes.reject { |k,v| k.to_s == 'id'}.to_json
+      omit_attrs = [ 'id' ]
+      omit_attrs += self.class.schema.attrs_to_omit_during_update if persisted?
+      @attributes.reject { |k,v| omit_attrs.include?(k.to_s) }.to_json
     end
 
     def load_attributes(attributes_and_values) # :nodoc:
@@ -361,13 +361,13 @@ module LucidWorks
           next # Dont overwrite our connection to our parent
         end
         unless self.class.schema.has_attribute?(attr)
-          if self.class.has_schema
-            raise "unknown attribute: \"#{attr}\""
-          else
+          if self.class.schema.dynamic_attributes?
             self.class.schema.add_attribute(self.class, attr, :string)
+          else
+            raise "unknown attribute: \"#{attr}\""
           end
         end
-        @attributes[attr] = value
+        send "#{self.class.schema.sanitize_identifier(attr)}=", value
       end
     end
 

data/lib/lucid_works/collection.rb

@@ -2,21 +2,81 @@ module LucidWorks
 
   class Collection < Base
 
-    self.primary_key = :name
-
-    has_many :datasources, :fields
+    has_many :datasources, :fields, :activities
     has_one  :info, :settings
     has_one  :index, :has_content => false
 
     schema do
-      attribute :name
+      attribute :name, :string, :primary_key => true
       attribute :instance_dir
     end
 
     validates_presence_of :name
-    
+
+    def destroyable?
+      # Don't let user destroy 'lwelogs'
+      instance_dir != 'lwelogs'
+    end
+
     def empty!
       index.destroy(:params => {:key => 'iaccepttherisk'})
     end
+
+    # Setup the Collection with an RSolr object that it can use to search.
+    # Must be provided with the URL of a Solr instance (excluding the /solr/... path)
+    def rsolr_connect(solr_url, default_search_params={})
+      @default_search_params = default_search_params
+      @path_prefix = URI.parse(solr_url).path
+      @rsolr ||= RSolr.connect :url => solr_url.dup
+    end
+
+    # Perform a Solr search using RSolr
+    def search(search_params={}, options={})
+      params = @default_search_params.merge(search_params)
+      page = options[:page] ||= 1
+      per_page = options[:per_page] ||= 10
+      resp = @rsolr.paginate page, per_page, "#{@path_prefix}/solr/#{name}/select", :params => params
+      if params[:wt] == :xml
+        data = Nokogiri.XML(resp)
+        raise "search received bad XML" unless data.root
+      else
+        data = resp
+      end
+      data
+    end
+
+    def prime_activities
+      self.activities!
+      num_created = 0
+      activities_to_return = %w(optimize spelling click autocomplete).map do |type|
+        if act = self.activities.detect{|act| act.type == type}
+          act
+        else
+          num_created += 1
+          self.create_activity(:type => type, :active => true, 'start_time' => 3600*num_created)
+        end
+      end
+      self.activities! if num_created > 0
+      activities_to_return
+    end
+
+    # return the first for each kind of activity
+    # don't use these if you need more than one activity
+    # b/c each forces an API hit
+    def optimize_activity
+      prime_activities.detect{|act| act.type == 'optimize'}
+    end
+
+    def spelling_activity
+      prime_activities.detect{|act| act.type == 'spelling'}
+    end
+
+    def click_activity
+      prime_activities.detect{|act| act.type == 'click'}
+    end
+
+    def autocomplete_activity
+      prime_activities.detect{|act| act.type == 'autocomplete'}
+    end
   end
 end

data/lib/lucid_works/collection/activity.rb

@@ -0,0 +1,33 @@
+module LucidWorks
+  class Collection < Base
+    class Activity < Base
+      TYPES = %w{ optimize spelling click autocomplete}
+
+      belongs_to :collection
+      has_many   :histories, :class_name => :history
+      has_one    :status
+
+      schema do
+        attributes :start_time, :period
+        attribute :active, :boolean
+        attribute :type, :string, :values => TYPES, :omit_during_update => true
+      end
+
+      validates_presence_of :type, :start_time
+      validates_numericality_of :period, :allow_blank => true
+
+      def t_type
+        I18n.t(type, :scope => 'activemodel.models.lucid_works.datasource.type')
+      end
+
+      def start
+        self.start_time = 0
+        self.active = true
+        save
+      end
+
+      private
+
+    end
+  end
+end

data/lib/lucid_works/collection/activity/history.rb

@@ -0,0 +1,20 @@
+module LucidWorks
+  class Collection::Activity
+    class History < Base
+      belongs_to :activity
+      self.collection_name = 'history' # i.e. not the plural 'histories'
+
+      def activity_finished
+        Time.iso8601 activityFinished
+      end
+
+      def activity_started
+        Time.iso8601 activityStarted
+      end
+
+      def duration
+        activity_finished - activity_started
+      end
+    end
+  end
+end

data/lib/lucid_works/collection/activity/status.rb

@@ -0,0 +1,14 @@
+module LucidWorks
+  class Collection::Activity
+    class Status < Base
+      self.singleton = true
+      belongs_to :activity
+
+      schema do
+        attribute :running, :boolean
+        attribute :type, :string, :values => LucidWorks::Collection::Activity::TYPES
+      end
+
+    end
+  end
+end