lucid_works 0.2.0 → 0.3.9

data/lib/lucid_works/base.rb

@@ -2,29 +2,35 @@ module LucidWorks
 
   # LucidWorks::Base is our REST ORM foundation.
   #
-  # The motivaiton for developing it was:
-  # * ActiveResource makes a lot of assumtions about how REST APIs should work that we had to keep patching it for
-  #   (e.g. parameters must always be nested inside :resouce => {}).
+  #   class Collection < LucidWorks::Base
+  #   end
   #
-  # * ActiveResource was missing some features we needed:
-  #   - send new id with POST
-  #   - associations
-  #   - multipart POST
+  # When creating or retrieving objects, a parent LucidWorks::Base or LucidWorks::Server object must be supplied, e.g.
   #
-  # * The straw that broke the camel's back was the need to talk to the same type of REST API on multiple servers
-  #   simultaneously, which was so orthogonal to the design of ActiveResource that patching it to support this would
-  #   require a major re-write.
+  #   server = LucidWorks::Server.new('http://localhost:8888')
   #
-  # So, I decided to marry ActiveModel and RestClient.
+  #   collection = Collection.find('collection1', :parent => server)
   #
-  # Much of the internal structure here (method names etc) will resemble ActiveResource as I have spent much time
-  # wandering through that code.
+  #   datasource = Datasource.find(1, :parent => collection)
+  #
+  # This mechanism is used to build up the URI that will be used to manipulate the object in the REST API.
+  # For the above example:
+  #
+  #   datasource.uri -> http://localhost:8888/api/collections/collection1/datasources/1
+  #
+  # If you are accessing models using associations, the parent argument is taken care of for you:
+  #
+  #   datasource = server.collection('collection1').datasource(1)
 
   class Base
+    # Much of the internal structure here (method names etc) will resemble ActiveResource as I have spent much time
+    # wandering through that code.
+
     include ActiveModel::Validations
     include ActiveModel::Conversion
-    extend ActiveModel::Naming
-
+    extend  ActiveModel::Naming
+    extend  ActiveModel::Translation
+    extend  ActiveModel::Callbacks
     include Associations
 
     attr_accessor :parent           # :nodoc:
@@ -33,17 +39,50 @@ module LucidWorks
     attr_accessor :raw_response     # :nodoc:
     attr_accessor :response_data    # :nodoc:
 
+    define_model_callbacks :save, :only => [:before, :after]
+
     class << self
+      include ActionView::Helpers::NumberHelper rescue nil
+
       attr_accessor_with_default :primary_key, :id
-      attr_accessor :collection_name
+      attr_accessor :collection_name # :nodoc:
       attr_accessor_with_default :singleton, false
+      attr_accessor_with_default :has_schema, false
+
+      # The attributes for a model are ascertained in on of two ways.
+      # Without a schema, the attributes list is automatically generated when the the object is retrieved from the server.
+      # Alternatively, you may define a schema for your object.
+      # Objects with attributes defined by schemas may have validations run against those attributes during creation.
+      #
+      # Schema should be passed a block:
+      #
+      #   class Collection < LucidWorks::Base
+      #     schema do
+      #       attribute :name
+      #     end
+      #
+      #     validates_presence_of :name
+      #   end
+
+      def schema(&block)
+        @schema ||= Schema.new
+        if block_given?
+          @schema.instance_eval(&block)
+          @schema.create_accessors_for_attributes(self)
+          self.has_schema = true
+        end
+        @schema
+      end
+
+      # Create a new model.
+      #
+      #   MyObject.create(attr => value, ..., :parent => server_or_object)
 
       def create(*arguments)
         new(*arguments).tap { |model| model.save }
       end
 
       # Retrieve one or more models from the server.
-      #
       # Find may be called in the following ways:
       #
       # Retrieve an entire collection:
@@ -57,10 +96,10 @@ module LucidWorks
       #   find(options)
       #   find(:singleton, options)
       #
-      # == Options
+      # === Options
       #
       # :parent - mandatory, another LucidWorks::Base instance or a LucidWorks::Server instance.
-      #
+
       def find(*arguments)
         unless arguments.first.is_a?(Symbol)
           # We weren't called with a symbol, figure out what kind of find this is and re-call
@@ -85,6 +124,7 @@ module LucidWorks
 
         parent = extract_parent_from_options(options)
         includes = options.delete(:include)
+        order = options.delete(:order)
 
         url = case kind_of_find
           when :all;       collection_url(parent)
@@ -92,7 +132,12 @@ module LucidWorks
           when :singleton; "#{parent.uri}/#{singleton_name}"
         end
 
-        raw_response = RestClient.get(url)
+        raw_response = ActiveSupport::Notifications.instrument("lucid_works.request") do |payload|
+          payload[:method]   = :get
+          payload[:uri]      = url
+          payload[:response] = RestClient.get(url)
+        end
+
         data = JSON.parse(raw_response)
 
         results =
@@ -119,9 +164,12 @@ module LucidWorks
           end
         end
 
+        results.sort! { |a,b| a.send(order) <=> b.send(order) } if order
         results
       end
 
+      # Shortcut for find(:all, options)
+
       def all(options)
         find(:all, options)
       end
@@ -134,6 +182,43 @@ module LucidWorks
         find(:all, options).last
       end
 
+      # Convert the attribute value to a string.  If a schema has been defined for the modeland a type has
+      # been defined for the attribute, it will have formatting applied as follows:
+      #
+      # - <tt>boolean</tt> will be converted to 'yes' or 'no'
+      # - <tt>integer</tt> will be passed to number_with_delimter
+      #
+      # If the attributes is listed in the schema as having :values =>, it will be translated.
+
+      def human_attribute_value(attribute, value)
+        if schema[attribute][:values]
+          l10n_scope = %w{activemodel models} + self.name.underscore.split('/') + [attribute]
+          return I18n.t(value, :scope => l10n_scope, :default => value)
+        end
+
+        case schema[attribute][:type]
+        when :boolean
+          value.to_yesno
+        when :integer
+          number_with_delimiter(value)
+        else
+          value.to_s
+        end
+      rescue
+        value.to_s
+      end
+
+      # For attributes listed in the schema as having :values, this will create an array-of-arrays
+      # suitable for use as options_for_select.  The
+
+      def to_select(attribute)
+        raise "Can't to_select for #{attribute} as it has no values" unless schema[attribute][:values]
+        l10n_scope = %w{activemodel models} + self.name.underscore.split('/') + [attribute]
+        schema[attribute][:values].map do |value|
+          [human_attribute_value(attribute, value), value]
+        end
+      end
+
       def collection_name # :nodoc:
         @collection_name || name.underscore.gsub(/^.*\//, '').pluralize
       end
@@ -166,29 +251,49 @@ module LucidWorks
       raise ArgumentError.new("new requires a Hash") unless options.is_a?(Hash)
       @parent = self.class.extract_parent_from_options(options)
       @persisted = options.delete(:persisted) || singleton? || false
-      @attributes = options.with_indifferent_access
+      @attributes = {}.with_indifferent_access
+      load_attributes(options)
     end
 
     def save
-      if valid?
-        begin
-          if persisted?
-            response = RestClient.put(member_url, encode, :content_type => :json)
-          else
-            response = RestClient.post(collection_url, encode, :content_type => :json)
-            @persisted = true
+      _run_save_callbacks do
+        if valid?
+          ActiveSupport::Notifications.instrument("lucid_works.request") do |payload|
+            method, uri = persisted? ? [:put, member_url] : [:post, collection_url]
+            data = encode
+            payload[:method] = method
+            payload[:uri] = uri
+            payload[:data] = data
+            begin
+              response = RestClient.send(method, uri, data, :content_type => :json)
+              payload[:response] = response
+              @persisted = true
+              load_attributes_from_json_string(response)
+              true
+            rescue RestClient::UnprocessableEntity, RestClient::Conflict => e
+              payload[:exception] = e
+              attach_errors_to_model(e.response)
+              false
+            end
           end
-          load_attributes_from_json_string(response)
-          true
-        rescue RestClient::UnprocessableEntity, RestClient::Conflict => e
-          attach_errors_to_model(e.response)
-          false
         end
       end
     end
-    
+
+    def update_attributes(attrs_and_values)
+      attrs_and_values.each do |attr,value|
+        self.send("#{attr}=", value)
+      end
+      save
+    end
+
     def destroy(options={})
-      RestClient.delete(member_url, options)
+      ActiveSupport::Notifications.instrument("lucid_works.request") do |payload|
+        payload[:method]   = :delete
+        payload[:uri]      = member_url
+        payload[:options]  = options
+        payload[:repsonse] = RestClient.delete(member_url, options)
+      end
     end
 
     def id # :nodoc:
@@ -203,21 +308,6 @@ module LucidWorks
       @persisted
     end
 
-    def method_missing(method_sym, *arguments) # :nodoc:
-      return super if method_sym == :to_ary
-      if method_sym.to_s =~ /^(\w+)=$/
-        return @attributes[$1] = arguments.first
-      elsif method_sym.to_s =~ /^(\w+)\?$/
-        attr = $1
-        predicate = true
-      else
-        attr = method_sym
-        predicate = false
-      end
-      raise "Unknown attribute: '#{attr}'" unless @attributes.has_key?(attr)
-      predicate ? !!@attributes[attr] : @attributes[attr]
-    end
-
     def read_attribute_for_validation(key) # :nodoc:
       @attributes[key]
     end
@@ -236,10 +326,17 @@ module LucidWorks
     
     alias :uri :member_url
 
-    def inspect
+    def inspect # :nodoc:
       "<#{self.class.name} " + @attributes.map { |k,v| "#{k}=#{v.inspect}" }.join(" ") + ">"
     end
 
+    # Convert the attribute value to a string.
+    # See LucidWorks::Base.human_attribute_values for details.
+        
+    def human_attribute_value(attribute)
+      self.class.human_attribute_value(attribute, send(attribute))
+    end
+
     private
 
     def singleton? # :nodoc:
@@ -254,12 +351,30 @@ module LucidWorks
       @attributes.reject { |k,v| k.to_s == 'id'}.to_json
     end
 
-    def load_attributes_from_json_string(response) # :nodoc:
-      data = JSON.parse(response) rescue {}
-      data.each do |k,v|
-        @attributes[k] = v
+    def load_attributes(attributes_and_values) # :nodoc:
+      attributes_and_values.each do |attr, value|
+        # Special cases - don't overwrite built-in accessors
+        if attr.to_sym == :id
+          self.id = value
+          next
+        elsif self.class.respond_to?(:belongs_to_association_name) && attr.to_sym == self.class.belongs_to_association_name
+          next # Dont overwrite our connection to our parent
+        end
+        unless self.class.schema.has_attribute?(attr)
+          if self.class.has_schema
+            raise "unknown attribute: \"#{attr}\""
+          else
+            self.class.schema.add_attribute(self.class, attr, :string)
+          end
+        end
+        @attributes[attr] = value
       end
     end
+
+    def load_attributes_from_json_string(response) # :nodoc:
+      attribute_data = JSON.parse(response) rescue {}
+      load_attributes(attribute_data)
+    end
     
     def attach_errors_to_model(response) # :nodoc:
       data = JSON.parse(response) rescue nil

data/lib/lucid_works/collection/info.rb

@@ -4,6 +4,19 @@ module LucidWorks
     class Info < Base
       self.singleton = true
       belongs_to :collection
+
+      schema do
+        attributes :free_disk_space, :total_disk_space, :index_size,
+                   :index_last_modified, :collection_name,
+                   :data_dir, :root_dir, :instance_dir,
+                   :type => :string
+        attribute  :index_directory # a hash
+        attributes :index_size_bytes, :free_disk_bytes, :total_disk_bytes,
+                   :index_max_doc, :index_num_docs, :index_version,
+                   :type => :integer
+        attributes :index_has_deletions, :index_is_optimized, :index_is_current,
+                   :type => :boolean
+      end
     end
   end
 end

data/lib/lucid_works/collection/settings.rb

@@ -4,6 +4,20 @@ module LucidWorks
     class Settings < Base
       self.singleton = true
       belongs_to :collection
+
+      DEDUP_OPTIONS = %w{ off overwrite tag }
+
+      schema do
+        attributes :unsupervised_feedback_emphasis, :unknown_type_handling,
+                   :click_boost_field, :click_boost_data, :query_parser, :default_sort,
+                   :type => :string
+        attribute  :de_duplication, :string, :values => DEDUP_OPTIONS
+        attributes :spellcheck, :display_facets, :ssl, :unsupervised_feedback, :query_time_stopwords,
+                   :auto_complete, :boost_recent, :click_enabled, :show_similar, :query_time_synonyms,
+                   :index_time_stopwords, :type => :boolean
+        attributes :search_server_list, :update_server_list, :stopword_list, :boosts, :synonym_list # Arrays
+        attribute :elevations # Hash
+      end
     end
   end
 end

data/lib/lucid_works/collection.rb

@@ -4,11 +4,19 @@ module LucidWorks
 
     self.primary_key = :name
 
-    has_many :datasources
-    has_one  :info, :settings, :index
+    has_many :datasources, :fields
+    has_one  :info, :settings
+    has_one  :index, :has_content => false
 
+    schema do
+      attribute :name
+      attribute :instance_dir
+    end
+
+    validates_presence_of :name
+    
     def empty!
-      build_index.destroy(:params => {:key => 'iaccepttherisk'})
+      index.destroy(:params => {:key => 'iaccepttherisk'})
     end
   end
 end

data/lib/lucid_works/crawler.rb

@@ -0,0 +1,8 @@
+module LucidWorks
+
+  class Crawler < Base
+
+    self.primary_key = :name
+
+  end
+end

data/lib/lucid_works/datasource/crawldata.rb

@@ -0,0 +1,9 @@
+module LucidWorks
+  class Datasource
+
+    class Crawldata < Base
+      self.singleton = true
+      belongs_to :datasource
+    end
+  end
+end

data/lib/lucid_works/datasource/history.rb

@@ -9,8 +9,16 @@ module LucidWorks
         numUpdated + numNew + numUnchanged
       end
 
-      def elapsed_time
-        crawlStopped.to_datetime - crawlStarted.to_datetime
+      def crawl_stopped
+        Time.iso8601 crawlStopped
+      end
+
+      def crawl_started
+        Time.iso8601 crawlStarted
+      end
+
+      def duration
+        crawl_stopped - crawl_started
       end
     end
   end

data/lib/lucid_works/datasource/schedule.rb

@@ -4,6 +4,13 @@ module LucidWorks
     class Schedule < Base
       self.singleton = true
       belongs_to :datasource
+
+      schema do
+        attribute :period
+        attribute :start_time
+        attribute :active
+        attribute :type
+      end
     end
   end
 end

data/lib/lucid_works/datasource/status.rb

@@ -4,6 +4,51 @@ module LucidWorks
     class Status < Base
       self.singleton = true
       belongs_to :datasource
+
+      schema do
+        attributes :crawlStarted, :crawlState, :crawlStopped, :jobId
+        attributes :numUnchanged, :numUpdated, :numNew, :numFailed, :numDeleted, :type => :integer
+      end
+
+      STOPPED_STATES =         %w{ IDLE STOPPED ABORTED EXCEPTION FINISHED }
+      POST_PROCESSING_STATES = %w{ STOPPING ABORTING }
+      CRAWLSTATES =            STOPPED_STATES + [ 'RUNNING' ] + POST_PROCESSING_STATES
+
+      # Create predicate methods for all the crawl states
+      CRAWLSTATES.each do |state|
+        method_name = state.downcase + "?"
+        class_eval <<-EOF
+          def #{method_name}
+            self.crawlState == "#{state}"
+          end
+        EOF
+      end
+
+      def stopped?
+        STOPPED_STATES.include?(crawlState)
+      end
+
+      def post_processing?
+        POST_PROCESSING_STATES.include?(crawlState)
+      end
+
+      def doc_count
+        numUpdated + numNew + numUnchanged
+      end
+
+      def t_crawl_state
+        I18n.translate(crawlState.downcase,
+                       :scope => 'activemodel.models.lucid_works.datasource.status.crawl_state',
+                       :default => crawlState)
+      end
+
+      def crawl_started
+        Time.iso8601 crawlStarted
+      end
+
+      def elapsed_time
+        Time.now - crawl_started
+      end
     end
   end
 end

data/lib/lucid_works/datasource.rb

@@ -3,38 +3,57 @@ module LucidWorks
   class Datasource < Base
     belongs_to :collection
     has_many   :histories, :class_name => :history
-    has_one    :status, :history, :schedule, :index
-
-    TYPES = {
-      :FileSystemDataSource => {
-        :name_l10n_key => 'data_source.short_type.filesystem',
-        :crawler       => 'lucid.aperture',
-        :type          => 'file'
-      },
-      :WebDataSource => {
-        :name_l10n_key => 'data_source.short_type.web_site',
-        :crawler       => 'lucid.aperture',
-        :type          => 'web'
-      },
-      :SolrXmlDataSource => {
-        :name_l10n_key => 'data_source.short_type.solr',
-        :crawler       => 'lucid.solrxml',
-        :type          => 'solrxml'
-      },
-      :JDBCDataSource => {
-        :name_l10n_key => 'data_source.short_type.database',
-        :crawler       => 'lucid.jdbc',
-        :type          => 'jdbc'
-      },
-      :SharePointDataSource => {
-        :name_l10n_key => 'data_source.short_type.sharepoint',
-        :crawler       => 'lucid.gcm',
-        :type          => 'sharepoint'
-      }
+    has_one :status, :schedule, :crawldata
+    has_one :index, :has_content => false
+
+    schema do
+      # common
+      attributes :name, :type, :crawler
+      attributes :crawl_depth, :max_bytes, :type => :integer
+      attribute :include_paths
+      attribute :exclude_paths
+      attribute :mapping # Hash
+      attribute :bounds
+      # web
+      attributes :url, :category
+      attribute :collect_links, :boolean
+      # file
+      attribute :path
+      attribute :follow_links, :boolean
+    end
+
+    validates_presence_of :type, :crawler, :name, :crawl_depth
+    validates_numericality_of :max_bytes, :allow_blank => true
+    validates_presence_of :url, :if => lambda { |d| d.type == 'web' }
+
+    before_save :remove_blank_max_bytes
+
+    TYPES = %w{ file web solrxml jdbc sharepoint }
+    BOUNDS = %w{ tree none }
+    CRAWLERS = {
+      # Later we may change these to be arrays if we decide to support more than one choice
+      # e.g. :web => ['lucid.aperture', 'nutch']
+      :file       => 'lucid.aperture',
+      :web        => 'lucid.aperture',
+      :solrxml    => 'lucid.solrxml',
+      :jdbc       => 'lucid.jdbc',
+      :sharepoint => 'lucid.gcm'
     }.with_indifferent_access
 
+
     def empty!
-      build_index.destroy
+      index.destroy
+    end
+
+    def t_type
+      I18n.t(type, :scope => 'activemodel.models.lucid_works.datasource.type')
+    end
+
+    private
+
+    def remove_blank_max_bytes # :nodoc:
+      # API can't handle a blank max_bytes.  Send nothing to select 'unlimited'
+      @attributes.delete :max_bytes if self.max_bytes.blank?
     end
   end
 end

data/lib/lucid_works/field.rb

@@ -0,0 +1,53 @@
+module LucidWorks
+
+  class Field < Base
+    belongs_to :collection
+
+    TYPES = [
+      'string',
+      'boolean',
+      'binary',
+      'int',
+      'float',
+      'long',
+      'double',
+      'tint',
+      'tfloat',
+      'tlong',
+      'tdouble',
+      'uri',
+      'date',
+      'tdate',
+      'text_ws',
+      'text_en',
+      'text_porter_en',
+      'textTight',
+      'text_cjk',
+      'text_da',
+      'text_de',
+      'text_es',
+      'text_fr',
+      'text_it',
+      'text_nl',
+      'text_pt',
+      'text_ru',
+      'text_se',
+      'text_fi',
+      'random',
+      'comma-separated',
+      'textSpell',
+      'payloads',
+      'point',
+      'location',
+      'geohash'
+    ]
+
+    def t_field_type
+      self.class.t_field_type(self.field_type)
+    end
+
+    def self.t_field_type(type)
+      I18n.translate(type, :scope => 'activemodel.models.lucid_works.collection.field.field_type')
+    end
+  end
+end

data/lib/lucid_works/logs.rb

@@ -2,7 +2,6 @@ module LucidWorks
   class Logs < LucidWorks::Base
     self.singleton = true
   
-    has_one :query
-    has_one :index
+    has_one :query, :index, :has_content => false
   end
 end