lucid_works 0.7.18 → 0.9.4

data/lib/lucid_works/cache.rb

@@ -0,0 +1,13 @@
+module LucidWorks
+  class Cache < Base
+    belongs_to :collection
+    
+    schema do
+      dynamic_attributes true
+      attribute :name, :string, :primary_key => true, :omit_during_update => true
+    end
+
+    validates_presence_of :size
+    validates_numericality_of :initial_size, :size, :only_integer => true, :greater_than_or_equal_to => 0, :allow_blank => true, :message => 'is not a valid value'
+  end
+end

data/lib/lucid_works/cluster.rb

@@ -0,0 +1,84 @@
+require 'uri'
+
+module LucidWorks
+  class Cluster
+    class Node
+      include ActiveModel::Serializers::JSON
+      self.include_root_in_json = false
+      
+      attr_reader :id, :cluster, :host, :port, :state, :solr_url, :api_url
+      attr_accessor :shards
+    
+      def initialize(cluster, solr_url, state)
+        @cluster, @solr_url, @state = cluster, solr_url, state
+        uri = URI(solr_url)
+        @host, @port = uri.host, uri.port
+        @id = "#{host}_#{port}"
+        @api_url = solr_url.sub uri.path, '/api'
+        @shards = []
+      end
+      
+      def as_json(options = {})
+        {'id' => id, 'host' => host, 'port' => port, 'state' => state, 'solr_url' => solr_url, 'api_url' => api_url, 'shards' => shards}
+      end
+    end
+    
+    class Shard
+      include ActiveModel::Serializers::JSON
+      self.include_root_in_json = false
+
+      attr_reader :id, :node, :name, :collection, :leader
+      alias :leader? :leader
+    
+      def initialize(node, name, collection, leader)
+        @id = "#{collection}_#{name}"
+        @node = node
+        @name = name
+        @collection = collection
+        @leader = leader
+      end
+
+      def as_json(options = {})
+        {'name' => name, 'collection' => collection, 'leader' => leader}
+      end
+      
+      def ==(other)
+        node == other.node && name == other.name && collection == other.collection
+      end
+    end
+    
+    attr_reader :nodes
+    
+    def initialize(solr_uri)
+      zk_uri = "#{solr_uri}/zookeeper?detail=true&path="
+      @zk_clusterstate = JSON.parse(JSON.parse(RestClient.get(zk_uri + "/clusterstate.json"))['znode']['data'])
+      @zk_live_nodes = JSON.parse(RestClient.get(zk_uri + "/live_nodes/"))['tree'].first['children'].
+        map {|child| child['data']['title'] } rescue []
+        
+      @nodes = []
+      @zk_clusterstate.each do |slice|
+        slice[1]['shards'].each do |shard_name, shard_description|
+          unless shard_description.empty? # A shard's description can come back as empty when the shard is in trouble
+            shard_description['replicas'].values.each do |node_json|
+              node = nodes.find {|node| node.solr_url == node_json['base_url'] }
+              nodes << (node = Node.new(self, node_json['base_url'], determine_node_state(node_json))) unless node
+              new_shard = Shard.new(node, node_json['shard'], node_json['collection'], node_json['leader'] == 'true')
+              node.shards << new_shard unless node.shards.include?(new_shard)
+            end
+          end
+        end
+      end
+    end
+    
+    private
+    
+      # State can be "live", "down", "recovering", "recovery_failed"
+      def determine_node_state(node_json)
+        if @zk_live_nodes.include?(node_json['node_name'])
+          node_json['state'] == 'active' ? 'live' : node_json['state']
+        else
+          'down'
+        end
+      end
+  end
+end

data/lib/lucid_works/collection/settings.rb

@@ -14,16 +14,17 @@ module LucidWorks
 
       schema do
         # Indexing Settings
-        attribute  :unknown_type_handling, :string
-        attribute  :de_duplication, :string, :values => DEDUP_OPTIONS
-
+        attribute :unknown_type_handling, :string, :nil_when_blank => true
+        attribute :de_duplication, :string, :values => DEDUP_OPTIONS
+        attributes :update_handler_autocommit_max_docs, :update_handler_autocommit_max_time, 
+          :update_handler_autosoftcommit_max_docs, :update_handler_autosoftcommit_max_time
+          
         # Querying Settings
         attribute :unsupervised_feedback_emphasis, :string, :values => FEEDBACK_EMPHASIS
         attribute :default_sort, :string, :values => DEFAULT_SORTS
         attribute :query_parser, :string, :values => QUERY_PARSERS
         attributes :spellcheck, :display_facets, :unsupervised_feedback, :query_time_stopwords,
-                   :auto_complete, :boost_recent, :show_similar, :query_time_synonyms,
-                   :type => :boolean
+         :auto_complete, :boost_recent, :show_similar, :query_time_synonyms, :type => :boolean
         attributes :stopword_list, :boosts, :synonym_list # Arrays
 
         # Click Settings
@@ -33,10 +34,18 @@ module LucidWorks
         # Other Settings
         attribute :ssl, :boolean
         attribute :elevations # Hash
+        attributes :main_index_ram_buffer_size_mb, :main_index_merge_factor
 
-        # Distrubuted Search Settings
+        # Distributed Search Settings
         attributes :search_server_list, :update_server_list # Arrays
       end
+      
+      validates_numericality_of :main_index_ram_buffer_size_mb, :main_index_merge_factor, 
+        :allow_blank => true, :message => 'is not a valid value'
+      validates_numericality_of :update_handler_autocommit_max_docs, :update_handler_autocommit_max_time, 
+        :update_handler_autosoftcommit_max_docs, :update_handler_autosoftcommit_max_time,
+        :only_integer => true, :greater_than => 0, :allow_blank => true,
+        :message => 'is not a valid value'
     end
   end
 end

data/lib/lucid_works/collection.rb

@@ -3,27 +3,30 @@ module LucidWorks
   class Collection < Base
 
     belongs_to :server
-    has_many :datasources, :fields, :fieldtypes, :activities, :roles
+    has_many :datasources, :fields, :fieldtypes, :activities, :roles, :caches, :dynamicfields
     has_one  :info, :settings, :click
     has_one  :index, :has_content => false
-
+    
     schema do
       attribute :name, :string, :primary_key => true
       attribute :instance_dir
       attribute :template
+      attribute :num_shards, :integer, :omit_when_blank => true
     end
 
     LOGS_COLLECTION_NAME = 'LucidWorksLogs'
-    AD_FILTERING = 'adfiltering'
-    ROLE_FILTERING = 'filterbyrole'
-    STATIC_ACL_CONFIG = {
-      "filterer.class" => "com.lucid.security.WindowsACLQueryFilterer",
-      "provider.class" => "com.lucid.security.ad.ADACLTagProvider",
-    }
-    MAGIC_ACL_ONLY_FILTER_SETTING = {'should_clause' => '*:* -data_source_type:smb'}
     
     validates_presence_of :name
-
+    validates_presence_of :num_shards, :if => lambda {|c| c.server.clustered? }
+    validates_numericality_of :num_shards, :only_integer => true, :greater_than => 0, :allow_blank => true, :message => 'is not a valid value'
+        
+    def update_caches_attributes(attributes)
+      attributes = attributes.with_indifferent_access
+      caches.each {|cache| cache.attributes = attributes[cache.name] if attributes.include?(cache.name) }
+      return false unless caches.all? &:valid?
+      caches.all? &:save
+    end
+    
     def destroyable?
       true
     end
@@ -36,43 +39,35 @@ module LucidWorks
     def empty!
       build_index.destroy(:params => {:key => 'iaccepttherisk'})
     end
-
-    # Sometimes we want to use the raw Rsolr class, instead of Rsolr::Ext
-    def rsolr
-      unless @rsolr
-        server_uri = self.server.host
-        @path_prefix = URI.parse(server_uri).path # The API key
-        @rsolr = RSolr.connect :url => server_uri.dup
-      end
-      @rsolr
+    
+    def commit
+      solr.commit
     end
-
-    def rsolr_ext
-      unless @rsolr_ext
-        server_uri = self.server.host
-        @path_prefix = URI.parse(server_uri).path # The API key
-        @rsolr_ext = RSolr::Ext.connect :url => server_uri.dup
+    
+    def search(solr_params, options = {})
+      ActiveSupport::Notifications.instrument('solr.lucid_works') do |payload|
+        path = options[:path] || 'select'
+        begin
+          result = if options[:page] && options[:per_page]
+            # Constructing the request here instead of using result.request[:uri] to avoid losing it on RSolr::Error::Http errors
+            payload[:request] = solr.build_paginated_request(options[:page], options[:per_page], path, :params => solr_params)[:uri].to_s
+            solr.paginate(options[:page], options[:per_page], path, :params => solr_params)
+          else
+            # Constructing the request here instead of using result.request[:uri] to avoid losing it on RSolr::Error::Http errors
+            payload[:request] = solr.build_request(path, :params => solr_params)[:uri].to_s
+            solr.get(path, :params => solr_params)
+          end
+          payload[:status] = result.response[:status]
+          result
+        rescue RSolr::Error::Http => error
+          payload[:status] = error.response[:status]
+          raise
+        end
       end
-      @rsolr_ext
     end
 
-    # Lower level access to rsolr, for things like autocomplete queries
-    def rsolr_get(path_suffix, search_params)
-      rsolr.get "#{@path_prefix}/solr/#{name}/#{path_suffix}", :params => search_params
-    end
-
-    # Perform a Solr search using RSolr
-    def search(search_params={})
-      search_params[:page] ||= 1
-      search_params[:per_page] ||= 10
-      resp = rsolr_ext.find "#{@path_prefix}/solr/#{name}/select", search_params
-      if search_params[:wt] == :xml
-        data = Nokogiri.XML(resp)
-        raise "search received bad XML" unless data.root
-      else
-        data = resp
-      end
-      data
+    def document_count
+      solr.get('select', :params => {:wt => :ruby, :q => "*:*", :rows => 0})['response']['numFound']
     end
 
     def synonyms
@@ -85,6 +80,22 @@ module LucidWorks
       Synonym.new(attributes.merge(:collection => self))
     end
     
+    def elevations
+      elevations = []
+      settings.elevations.each do |query, elevation_entries|
+        elevation_entries.each do |entry|
+          elevations << Elevation.new(
+            :doc_id => entry['doc'], :collection => self, :query => query, :excluded => entry['exclude'], :persisted => true
+          )
+        end
+      end
+      elevations
+    end
+    
+    def build_elevation(attributes = {})
+      Elevation.new(attributes.merge(:collection => self))
+    end
+    
     def prime_activities
       self.activities!.sort!{|a,b|a.id <=> b.id}
       num_created = 0
@@ -115,7 +126,7 @@ module LucidWorks
 
     # URL of Solr's build-in admin page
     def admin_url
-      "#{server.host}/solr/#{name}/admin/"
+      "#{server.server_uri}/solr/#{name}/admin/"
     end
 
     # We have to handle the jdbcdrivers model in a custom way, as a GET on .../jdbcdrivers returns a list of strings,
@@ -136,59 +147,18 @@ module LucidWorks
       JSON.parse(RestClient.get(uri + "/components/all.json?handlerName=%2Flucid"))
     end
     
-    def filtering_enabled?
-      # current core implementation requires exactly one of filterbyrole or adfiltering
-      assert_components_include_ad_xor_role
-      return self.components.include?(AD_FILTERING)
-    end
-    
-    def acl_only?
-      filterer_config = self.filtering_settings['filterer.config']
-      return false if filterer_config.nil?
-      return self.filtering_settings['filterer.config'] != MAGIC_ACL_ONLY_FILTER_SETTING #rescue false
+    def available_templates
+      JSON.parse(RestClient.get(server.uri + "/collectiontemplates"))
     end
 
-    def assert_components_include_ad_xor_role
-      # require 'ruby-debug'; debugger
-      raise "conflicting filtering components" if self.components.include?(AD_FILTERING) && self.components.include?(ROLE_FILTERING)
-      raise "missing filtering components" if ! self.components.include?(AD_FILTERING) && ! self.components.include?(ROLE_FILTERING)
+    def request_handler(name)
+      RequestHandler.new(self, name)
     end
 
-    def filtering_settings
-      JSON.parse(RestClient.get(uri + "/filtering"))['adfiltering'] || {}
-    end
+    protected
 
-    def compute_component_set(acl_filtering_enabled)
-      new_component_set = self.components.clone
-      new_component_set.delete(ROLE_FILTERING)
-      new_component_set.delete(AD_FILTERING)
-      new_component_set.unshift acl_filtering_enabled == 'true' ? AD_FILTERING : ROLE_FILTERING
-    end
-    
-    def configure_filtering(opts)
-      if ! opts[:config]['java.naming.provider.url'].blank? && opts[:config]['java.naming.provider.url'] !~ %r(://)
-        opts[:config]['java.naming.provider.url'] = "ldap://#{opts[:config]['java.naming.provider.url']}" 
+      def solr
+        @solr ||= RSolr.connect(:url => "#{server.server_uri}/solr/#{name}")
       end
-      filtering_settings = STATIC_ACL_CONFIG.merge('provider.config' => opts[:config])
-      filtering_settings["filterer.config"] = opts[:acl_only] ? {} : MAGIC_ACL_ONLY_FILTER_SETTING
-      
-      errors = {}
-      method = RestClient.send(:get, uri+'/filtering')['adfiltering'] ? :put : :post
-      begin
-        response = RestClient.send(method, uri+'/filtering/adfiltering', filtering_settings.to_json, :content_type => :json)
-      rescue => exception
-        JSON.parse(exception.response)['errors'].each {|e| errors[e['code']] = e['message']}
-      end
-      
-      new_component_set = compute_component_set(opts[:enabled])
-      if new_component_set.sort != self.components.sort
-        begin
-          response = RestClient.send(:put, uri+'/components/components?handlerName=/lucid', new_component_set.to_json, :content_type => :json)
-        rescue => exception
-          JSON.parse(exception.response)['errors'].each {|e| errors[e['code']] = e['message']}
-        end
-      end
-      raise LucidWorks::AclConfigInvalid.new(errors) unless errors.empty?
-    end
   end
 end

data/lib/lucid_works/datasource/history.rb

@@ -6,7 +6,8 @@ module LucidWorks
       self.collection_name = 'history' # i.e. not the plural 'histories'
       schema do
         attributes :crawl_started, :crawl_stopped, :type => :iso8601
-        attributes :num_updated, :num_new, :num_unchanged, :type => :integer
+        attributes :num_updated, :num_new, :num_unchanged, :num_not_found, 
+          :num_filter_denied, :num_access_denied, :num_robots_denied, :type => :integer
       end
 
       def doc_count

data/lib/lucid_works/datasource/mapping.rb

@@ -0,0 +1,12 @@
+module LucidWorks
+  class Datasource
+    class Mapping < Base
+      self.singleton = true
+      belongs_to :datasource
+
+      schema do
+        attribute :datasource_field, :string, :omit_when_blank => true
+      end
+    end
+  end
+end

data/lib/lucid_works/datasource/schedule.rb

@@ -31,6 +31,7 @@ module LucidWorks
       #
       def frequency
         case period
+        when 1.minute.seconds..59.minutes.seconds then 'every'
         when 1.weeks.seconds then 'weekly'
         when 1.days.seconds then 'daily'
         when 1.hours.seconds then 'hourly'
@@ -44,6 +45,7 @@ module LucidWorks
       #
       def frequency=(frequency)
         self.period = case frequency
+        when 'every' then period
         when 'hourly' then 1.hours.seconds.to_i
         when 'daily' then 1.days.seconds.to_i
         when 'weekly' then 1.weeks.seconds.to_i
@@ -57,7 +59,6 @@ module LucidWorks
       #
       def next_start
         return start_time if (now = Time.now) <= start_time 
-        # require 'ruby-debug'; debugger
         time_since_start = now - start_time
         last_interval_num = (time_since_start / period).to_i
         next_interval_num =  if (time_since_start % period) == 0
@@ -111,8 +112,10 @@ module LucidWorks
         self.frequency = all_attributes['frequency']
         self.start_time = 
           case all_attributes['frequency']
+          when 'every'
+            self.period = all_attributes['period'].to_i
+            now.ceil(period)
           when 'weekly'
-            # require 'ruby-debug'; debugger
             start = now.beginning_of_week.advance(all_attributes['start'])
             start < now ? start.advance(:weeks => 1) : start 
           when 'daily'                                                       

data/lib/lucid_works/datasource/status.rb

@@ -5,7 +5,7 @@ module LucidWorks
       self.singleton = true
       belongs_to :datasource
 
-      STOPPED_STATES =         %w{ IDLE STOPPED ABORTED EXCEPTION FINISHED }
+      STOPPED_STATES =         %w{ IDLE STOPPED ABORTED EXCEPTION FINISHED UNKNOWN }
       POST_PROCESSING_STATES = %w{ STOPPING ABORTING }
       CRAWLSTATES =            STOPPED_STATES + [ 'RUNNING' ] + POST_PROCESSING_STATES
 
@@ -13,7 +13,8 @@ module LucidWorks
         attribute :crawl_state, :string, :values => CRAWLSTATES
         attribute :jobId
         attributes :crawl_started, :crawl_stopped, :type => :iso8601
-        attributes :num_unchanged, :num_updated, :num_new, :num_failed, :num_deleted, :num_total, :type => :integer
+        attributes :num_unchanged, :num_updated, :num_new, :num_failed, :num_deleted, :num_total,
+          :num_not_found, :num_filter_denied, :num_access_denied, :num_robots_denied, :type => :integer
       end
 
       # Create predicate methods for all the crawl states

data/lib/lucid_works/datasource.rb

@@ -4,51 +4,33 @@ module LucidWorks
     belongs_to :collection
     has_many   :histories, :class_name => :history, :retrieveable_en_masse => true
     has_one :status, :retrieveable_en_masse => true
-    has_one :schedule, :crawldata
+    has_one :schedule, :crawldata, :mapping
     has_one :index, :job, :has_content => false
 
-    TYPES = %w{ external file ftp hdfs kfs lucidworkslogs web s3 s3n smb solrxml jdbc sharepoint }
-    SYSTEM_TYPES = %w{ lucidworkslogs }
     BOUNDS = %w{ tree none }
-    CRAWLERS = {
-      # Later we may change these to be arrays if we decide to support more than one choice
-      # e.g. :web => ['lucid.aperture', 'nutch'], :file => ['lucid.aperture', 'lucid.fs']
-      :file           => 'lucid.aperture',
-      :lucidworkslogs => 'lucid.logs',
-      :external       => 'lucid.external',
-      :web            => 'lucid.aperture',
-      :solrxml        => 'lucid.solrxml',
-      :jdbc           => 'lucid.jdbc',
-      :sharepoint     => 'lucid.gcm',
-      :ftp            => 'lucid.fs',
-      :hdfs           => 'lucid.fs',
-      :kfs            => 'lucid.fs',
-      :smb            => 'lucid.fs',
-      :s3n            => 'lucid.fs',
-      :s3             => 'lucid.fs'
-    }.with_indifferent_access
-
+    
     schema do
       # common
       attributes :name, :crawler
-      attribute :type, :string, :values => TYPES
+      attribute :type, :string
       attribute :crawl_depth, :integer, :nil_when_blank => true
       attribute :max_docs, :integer
-      attributes :max_bytes,   :commit_within, :type => :integer, :omit_when_blank => true
-      attribute :commit_within_min, :custom
+      attributes :max_bytes, :commit_within, :type => :integer, :omit_when_blank => true
+      attribute :commit_within_sec, :custom
       attribute :commit_on_finish, :boolean
-      attributes :include_paths, :exclude_paths, :type => :list, :separator => "\n"
-      attribute :mapping, :string, :omit_when_blank => true # Hash
+      attributes :include_paths, :exclude_paths, :filter_follow, :filter_track, :filter_locations,
+        :type => :list, :separator => "\n"
       attribute :bounds, :string, :values => BOUNDS
       # web
       attributes :url, :category
       attribute :collect_links, :boolean
       attribute :auth, :string, :omit_when_blank => true # Hash
-      attributes :proxy_host, :string, :omit_when_blank => true
+      attribute :proxy_host, :string, :omit_when_blank => true
       attribute :proxy_port, :string, :omit_when_blank => true
       attribute :proxy_username, :string, :omit_when_blank => true
       attribute :proxy_password, :string, :omit_when_blank => true
       attribute :ignore_robots, :boolean
+      attribute :add_failed_docs, :boolean
       # file
       attribute :path
       attribute :follow_links, :boolean
@@ -67,11 +49,24 @@ module LucidWorks
       attribute :password
       attribute :domain
       attribute :my_site_base_url, :string, :nil_when_blank => true
-      attribute :included_urls
-      attribute :excluded_urls
+      attributes :included_urls, :excluded_urls, :type => :list, :separator => "\n"
       attribute :kdcserver
       attribute :use_sp_search_visibility, :boolean
       attribute :aliases
+      attribute :feed_unpublished_documents, :boolean
+      attribute :push_acls, :boolean
+      attribute :enable_security_trimming, :boolean
+      attribute :username_format_in_ace
+      attribute :groupname_format_in_ace
+      attribute :ldap_server_host_address
+      attribute :ldap_server_port_number
+      attribute :ldap_server_use_ssl, :boolean
+      attribute :ldap_auth_type
+      attribute :ldap_search_base
+      attribute :ldap_read_ad_groups_type
+      attribute :ldap_cache_groups_membership, :boolean
+      attribute :ldap_cache_size
+      attribute :ldap_cache_refresh_interval
       # external
       attribute :source
       attribute :source_type
@@ -97,25 +92,22 @@ module LucidWorks
     validates_numericality_of :max_bytes, :allow_blank => true
     validates_presence_of :url, :if => lambda { |d| d.type == 'web' }
 
-    # Fake attributes to ease UI implementation
-    def commit_within_min
-      commit_within.blank? ? nil : commit_within / 1.second.milliseconds / 1.minute.seconds
+    # Virtual attributes to ease UI implementation
+    def commit_within_sec
+      commit_within.blank? ? nil : commit_within / 1.second.milliseconds
     end
-    def commit_within_min=(mins)
-      self.commit_within = mins.blank? ? nil : mins.to_i.minutes.milliseconds
+    def commit_within_sec=(seconds)
+      self.commit_within = seconds.blank? ? nil : seconds.to_i.seconds.milliseconds
     end
     def delete_after_days
-      delete_after.blank? ? nil : deleteAfter / 1.second.milliseconds / 1.day.seconds
+      delete_after.blank? ? nil : delete_after / 1.second.milliseconds / 1.day.seconds
     end
     def delete_after_days=(days)
       self.delete_after = days.blank? ? nil : days.to_i.days.milliseconds
     end
 
     def document_count
-      collection.rsolr_get(:select,
-                           :wt => :ruby,
-                           :q => (type == 'lucidworkslogs' ? "*:*" : "data_source:#{id}"),
-                           :rows => 0)['response']['numFound']
+      collection.search(:wt => :ruby, :q => (type == 'lucidworkslogs' ? "*:*" : "data_source:#{id}"), :rows => 0)['response']['numFound']
     end
 
     def empty!
@@ -132,11 +124,6 @@ module LucidWorks
       !collection.system?
     end
 
-    def crawlable?
-      # Don't let user schedule crawl of external datasources
-      !%w{ external }.include?(type)
-    end
-
     def start_crawl!
       build_job.save
     end
@@ -145,10 +132,6 @@ module LucidWorks
       build_job.destroy
     end
     
-    def t_type
-      I18n.t(type, :scope => 'activemodel.models.lucid_works.datasource.type')
-    end
-
     def progress
       return nil if status.stopped?
       return nil unless histories.size > 0

data/lib/lucid_works/datasource_property.rb

@@ -5,6 +5,7 @@ module LucidWorks
     attr_reader :description, :name, :allowed_values, :type, :default_value, :required, :read_only, :advanced
     alias :read_only? :read_only
     alias :advanced? :advanced
+    alias :required? :required
 
     def initialize(attributes = {})
       @description    = attributes['description']
@@ -14,7 +15,7 @@ module LucidWorks
       @default_value  = attributes['default_value']
       @required       = attributes['required']
       @read_only      = attributes['read_only']
-      @advanced       = attributes['advanced']
+      @advanced       = attributes['hints'].include? 'advanced' rescue false
     end
   end
 end

data/lib/lucid_works/datasource_type.rb

@@ -8,6 +8,8 @@ module LucidWorks
       "---"          # Separator UI hint
     ]
 
+    SYSTEM_TYPES = %w{ lucidworkslogs }
+
     attr_reader :crawler, :category, :type, :props
 
     def initialize(crawler, attributes = {})
@@ -17,9 +19,21 @@ module LucidWorks
       @props    = attributes['props']
     end
 
+    def system?
+      SYSTEM_TYPES.include? type
+    end
+
     def properties
       @properties ||= @props.map do |prop|
         DatasourceProperty.new(prop)
+      end.tap do |properties|
+        def properties.regular
+          select {|p| !p.advanced? }
+        end
+
+        def properties.advanced
+          select &:advanced?
+        end
       end
     end
 

data/lib/lucid_works/dynamicfield.rb

@@ -0,0 +1,12 @@
+require 'lucid_works/field_commons'
+
+module LucidWorks
+
+  class Dynamicfield < Base
+    include LucidWorks::FieldCommons
+
+    validates_each :name, :unless => :persisted?, :allow_blank => true do |model, attr, value|
+      model.errors.add(attr, 'must be unique') if model.collection.dynamicfields.any? {|f| f.name == value }
+    end
+  end
+end