lucid_works 0.2.0 → 0.3.9

data/lib/lucid_works/schema.rb

@@ -0,0 +1,72 @@
+module LucidWorks
+
+  class Schema < ActiveSupport::HashWithIndifferentAccess
+
+    # Specify an attribute for this model.
+    
+    def attribute(name, type=:string, options={})
+      self[name] = options.merge(:type => type)
+    end
+
+    # Specify multiple attributes.  Options apply to all attributes, e.g.
+    #
+    #   schema do
+    #     attributes :first_name, :last_name, :type => :string
+    #   end
+
+    def attributes(*attributes_and_options)
+      options = attributes_and_options.last.is_a?(Hash) ? attributes_and_options.pop : {}
+      attributes = attributes_and_options
+      attributes.each do |name|
+        self[name] = { :type => options[:type] || :string }
+      end
+    end
+
+    def has_attribute?(name)
+      has_key? sanitize_identifier(name)
+    end
+
+    # Used for classes that have no predefined schema.
+    # When the class is retrieved.
+    def add_attribute(klass, name, type=:string) # :nodoc:
+      attr = sanitize_identifier(name)
+      raise "Class #{klass.name} already has attribute #{attr}" if has_attribute?(attr)
+      self[attr] = { :type => type }
+      create_attribute(klass, attr)
+    end
+
+    # Create accessors for all attributes defined in the schema.
+    def create_accessors_for_attributes(klass) # :nodoc:
+      self.keys.each do |attr|
+        create_attribute(klass, attr)
+      end
+    end
+
+    def create_attribute(klass, attribute) # :nodoc:
+      klass.class_eval <<-EOF, __FILE__, __LINE__+1
+        def #{attribute}                            # def foo
+          @attributes[:#{attribute}]                #   @attributes[:foo]
+        end                                         # end
+
+        def #{attribute}=(new_value)                # def foo=(new_value)
+          @attributes[:#{attribute}] = new_value    #   @attributes[:foo] = new_value
+        end                                         # end
+      EOF
+      
+      if self[attribute][:type] == :boolean
+        klass.class_eval <<-EOF, __FILE__, __LINE__+1
+          def #{attribute}?                           # def foo?
+            @attributes[:#{attribute}]                #   @attributes[:foo]
+          end                                         # end
+        EOF
+      end
+    end
+
+    private
+
+     # Change any characters illegal for an identifier to _
+    def sanitize_identifier(identifier) # :nodoc:
+      identifier.to_s.gsub(/[^\w]/, '_').to_sym
+    end
+  end
+end

data/lib/lucid_works/server.rb

@@ -5,8 +5,8 @@ module LucidWorks
   class Server
     include Associations
 
-    has_many :collections
-    has_one :logs
+    has_many :collections, :crawlers
+    has_one :logs, :has_content => false
 
     DEFAULT_REST_API_PATH = "/api"
 

data/lib/lucid_works/version.rb

@@ -1,3 +1,3 @@
 module LucidWorks
-  VERSION = "0.2.0"
+  VERSION = "0.3.9"
 end

data/lib/lucid_works.rb

@@ -1,3 +1,5 @@
+# This is the namespace module in which all our classes and modules live.
+
 module LucidWorks
 end
 
@@ -5,6 +7,10 @@ require 'active_model'
 require 'active_support/core_ext/module/attr_accessor_with_default'
 require 'active_support/core_ext/hash/indifferent_access'
 require 'active_support/inflector'
+begin
+  require 'action_view/helpers/number_helper' # Optional formatter support
+rescue LoadError
+end
 require 'restclient'
 require 'json'
 
@@ -14,18 +20,24 @@ require 'lucid_works/exceptions'
 require 'lucid_works/associations'
 require 'lucid_works/server'
 require 'lucid_works/base'
+require 'lucid_works/schema'
 
 require 'lucid_works/collection'
 require 'lucid_works/collection/info'
 require 'lucid_works/collection/index'
 require 'lucid_works/collection/settings'
+require 'lucid_works/crawler'
 require 'lucid_works/datasource'
 require 'lucid_works/datasource/index'
 require 'lucid_works/datasource/status'
 require 'lucid_works/datasource/history'
 require 'lucid_works/datasource/schedule'
+require 'lucid_works/datasource/crawldata'
+require 'lucid_works/field'
 require 'lucid_works/logs'
 require 'lucid_works/logs/query'
 require 'lucid_works/logs/query/summary'
 require 'lucid_works/logs/index'
 require 'lucid_works/logs/index/summary'
+
+I18n.load_path += Dir.glob(File.join(File.dirname(__FILE__), '..', 'config', 'locales', '*.yml'))

data/spec/lib/lucid_works/associations_spec.rb

@@ -8,6 +8,7 @@ describe LucidWorks::Associations do
     class ::Blog < LucidWorks::Base
       has_many :posts
       has_one :homepage
+      has_one :launch_party, :has_content => false
     end
     @blog = ::Blog.new(:parent => @server)
 
@@ -19,30 +20,56 @@ describe LucidWorks::Associations do
       self.singleton = true
       belongs_to :blog
     end
+
+    class ::LaunchParty < LucidWorks::Base
+      self.singleton = true
+      belongs_to :blog
+    end
   end
 
   describe ".has_one" do
-    describe "#<child>!" do
-      it "should call Child.find" do
-        Homepage.should_receive(:find).with(:parent => @blog)
-        @blog.homepage!
+    context "without content" do
+      describe "#<child>" do
+        it "should build a new model, and not call REST API to retrieve" do
+          LaunchParty.should_not_receive(:find)
+          launch_party = @blog.launch_party
+
+          launch_party.should be_a(LaunchParty)
+          launch_party.should be_persisted # All singletons are always persisted
+        end
       end
     end
 
-    describe "#<child>" do
-      it "should call child! the first time then return the cached value thereafter" do
-        mock_homepage = double('homepage')
-        Homepage.should_receive(:find).once.and_return(mock_homepage)
-        @blog.homepage.should == mock_homepage
-        @blog.homepage.should == mock_homepage
+    context "with content" do
+      describe "#<child>!" do
+        it "should call Child.find" do
+          Homepage.should_receive(:find).with(:parent => @blog)
+          @blog.homepage!
+        end
+      end
+
+      describe "#<child>" do
+        it "should call child! the first time then return the cached value thereafter" do
+          mock_homepage = double('homepage')
+          Homepage.should_receive(:find).once.and_return(mock_homepage)
+          @blog.homepage.should == mock_homepage
+          @blog.homepage.should == mock_homepage
+        end
+      end
+
+      describe "#build_<child>" do
+        it "should create a new child with persisted = true" do
+          homepage = @blog.build_homepage
+          homepage.should be_a(Homepage)
+          homepage.should be_persisted
+        end
       end
     end
+  end
 
-    describe "#build_<child>" do
+  describe ".has_singleton" do
+    describe "#<child>" do
       it "should create a new child with persisted = true" do
-        homepage = @blog.build_homepage
-        homepage.should be_a(Homepage)
-        homepage.should be_persisted
       end
     end
   end

data/spec/lib/lucid_works/base_spec.rb

@@ -6,19 +6,100 @@ describe LucidWorks::Base do
     @server = LucidWorks::Server.new(fake_server)
     @fake_server_uri = "#{fake_server}/api"
 
+    @widget1 = { :id => 1, :name => 'wrench', :size => 'small' }
+    @widget2 = { :id => 2, :name => 'hammer', :size => 'medium' }
+    @widget3 = { :id => 3, :name => 'spanner', :size => 'large' }
+    @widgets = [@widget1, @widget2, @widget3]
+    WIDGET1_JSON = @widget1.to_json
+    WIDGET2_JSON = @widget2.to_json
+    WIDGETS_JSON = @widgets.to_json
+
     class Widget < LucidWorks::Base
       has_one :singleton_widget
     end
-    WIDGET1_JSON = '{"id":1,"name":"widget1","size":"small"}'
-    WIDGET2_JSON = '{"id":2,"name":"widget2","size":"medium"}'
-    WIDGETS_JSON = "[#{WIDGET1_JSON},#{WIDGET2_JSON}]"
 
     class SingletonWidget < LucidWorks::Base
       self.singleton = true
     end
+
+    class WidgetWithoutSchema < LucidWorks::Base
+    end
+
+    class WidgetWithSchema < LucidWorks::Base
+      schema do
+        attribute :name
+        attribute :frooble
+        attribute :iambool, :boolean
+      end
+    end
   end
 
   describe "class methods" do
+
+    describe ".schema" do
+      context "for a class with a schema" do
+        it "should set has_schema" do
+          WidgetWithSchema.has_schema.should be_true
+        end
+
+        it "should have accessors for the attributes defined in the schema" do
+          [:name, :name=, :frooble, :frooble=].each do |method|
+            WidgetWithSchema.new(:parent => @server).should respond_to(method)
+          end
+        end
+
+        it "should create predicate reader for boolean attributes" do
+          WidgetWithSchema.new(:parent => @server).should respond_to(:iambool?)
+        end
+
+        it "should remember the attribute values" do
+          widget = WidgetWithSchema.new(:parent => @server)
+          widget.frooble = "foo"
+          widget.frooble.should == "foo"
+        end
+
+        it "should raise an error for an unknown attribute" do
+          widget = WidgetWithSchema.new(:parent => @server)
+          lambda {
+            widget.unknown_attribute
+          }.should raise_error NoMethodError
+        end
+      end
+
+      context "for a class without a schema" do
+        it "should return false for has_schema" do
+          WidgetWithoutSchema.has_schema.should be_false
+        end
+
+        it "should create accessors upon retrieval" do
+          widget = WidgetWithoutSchema.new(:parent => @server)
+          widget.should_not respond_to(:name)
+          widget.should_not respond_to(:size=)
+          RestClient.should_receive(:get).with("#{@fake_server_uri}/widget_without_schemas/1").and_return(WIDGET1_JSON)
+          widget = WidgetWithoutSchema.find(1, :parent => @server)
+          widget.should respond_to(:name)
+          widget.should respond_to(:size=)
+        end
+
+        it "should remember the attribute values" do
+          class Widget2WithoutSchema < LucidWorks::Base ; end # Define another to stop bleedover from the above test
+          RestClient.should_receive(:get).with("#{@fake_server_uri}/widget2_without_schemas/1").and_return(WIDGET1_JSON)
+          widget = Widget2WithoutSchema.find(1, :parent => @server)
+          widget.size = 71077345
+          widget.size.should == 71077345
+        end
+
+        it "should raise an error for an unknown attribute" do
+          class Widget3WithoutSchema < LucidWorks::Base ; end # Define another to stop bleedover from the above test
+          RestClient.should_receive(:get).with("#{@fake_server_uri}/widget3_without_schemas/1").and_return(WIDGET1_JSON)
+          widget = Widget3WithoutSchema.find(1, :parent => @server)
+          lambda {
+            widget.unknown_attribute
+          }.should raise_error NoMethodError
+        end
+      end
+    end
+
     describe ".new" do
       it "should require a hash argument" do
         lambda {
@@ -97,17 +178,24 @@ describe LucidWorks::Base do
           RestClient.should_receive(:get).with(correct_url).and_return(WIDGETS_JSON)
 
           widgets = Widget.find(:all, :parent => @server)
-          widgets.size.should == 2
+          widgets.size.should == @widgets.size
 
-          widget1 = widgets.find { |c| c.id == 1 }
+          widget1 = widgets.find { |c| c.id == @widget1[:id] }
           widget1.should be_a(Widget)
-          widget1.name.should == 'widget1'
-          widget1.size.should == 'small'
+          widget1.name.should == @widget1[:name]
+          widget1.size.should == @widget1[:size]
 
-          widget2 = widgets.find { |c| c.id == 2 }
+          widget2 = widgets.find { |c| c.id == @widget3[:id] }
           widget2.should be_a(Widget)
-          widget2.name.should == 'widget2'
-          widget2.size.should == 'medium'
+          widget2.name.should == @widget3[:name]
+          widget2.size.should == @widget3[:size]
+        end
+
+        it "should sort items if given option :order" do
+          RestClient.stub(:get) { WIDGETS_JSON }
+          Widget.find(:all, :parent => @server, :order => :id).map(&:id).should == [1,2,3]
+          Widget.find(:all, :parent => @server, :order => :name).map(&:name).should == %w{ hammer spanner wrench }
+          Widget.find(:all, :parent => @server, :order => :size).map(&:size).should == %w{ large medium small }
         end
       end
 
@@ -116,10 +204,18 @@ describe LucidWorks::Base do
           correct_url = "#{@fake_server_uri}/widgets/1"
           RestClient.should_receive(:get).with(correct_url).and_return(WIDGET1_JSON)
 
-          widget = Widget.find(:one, 1, :parent => @server)
+          widget = Widget.find(:one, @widget1[:id], :parent => @server)
           widget.should be_a(Widget)
-          widget.name.should == 'widget1'
-          widget.size.should == 'small'
+          widget.name.should == @widget1[:name]
+          widget.size.should == @widget1[:size]
+        end
+
+        it "should sanitize any keys that cannot be represented as a Ruby identifier" do
+          RestClient.should_receive(:get).
+            with("#{@fake_server_uri}/widgets/1").
+            and_return('{"id":1,"bad.key":"period should map to underscore"}')
+          widget = Widget.find(:one, 1, :parent => @server)
+          widget.bad_key == "period should map to underscore"
         end
       end
 
@@ -130,8 +226,8 @@ describe LucidWorks::Base do
 
           widget = SingletonWidget.find(:singleton, :parent => @server)
           widget.should be_a(SingletonWidget)
-          widget.name.should == 'widget1'
-          widget.size.should == 'small'
+          widget.name.should == @widget1[:name]
+          widget.size.should == @widget1[:size]
         end
       end
 
@@ -141,12 +237,14 @@ describe LucidWorks::Base do
             RestClient.should_receive(:get).with("#{@fake_server_uri}/widgets").and_return(WIDGETS_JSON)
             mock_singleton1 = double('singleton widget1')
             mock_singleton2 = double('singleton widget2')
-            SingletonWidget.should_receive(:find).and_return(mock_singleton1, mock_singleton2)
+            mock_singleton3 = double('singleton widget3')
+            SingletonWidget.should_receive(:find).and_return(mock_singleton1, mock_singleton2, mock_singleton3)
 
             widgets = Widget.find(:all, :include => :singleton_widget, :parent => @server)
 
-            widgets.first.singleton_widget.should == mock_singleton1
-            widgets.last.singleton_widget.should == mock_singleton2
+            widgets[0].singleton_widget.should == mock_singleton1
+            widgets[1].singleton_widget.should == mock_singleton2
+            widgets[2].singleton_widget.should == mock_singleton3
           end
         end
 
@@ -161,17 +259,21 @@ describe LucidWorks::Base do
             RestClient.should_receive(:get).with("#{@fake_server_uri}/widgets").and_return(WIDGETS_JSON)
             mock_singleton1 = double('singleton widget1')
             mock_singleton2 = double('singleton widget2')
+            mock_singleton3 = double('singleton widget3')
             mock_other_widgets_1 = double('other widgets 1')
             mock_other_widgets_2 = double('other widgets 2')
-            SingletonWidget.should_receive(:find).and_return(mock_singleton1, mock_singleton2)
-            OtherWidget.should_receive(:find).and_return(mock_other_widgets_1, mock_other_widgets_2)
+            mock_other_widgets_3 = double('other widgets 3')
+            SingletonWidget.should_receive(:find).and_return(mock_singleton1, mock_singleton2, mock_singleton3)
+            OtherWidget.should_receive(:find).and_return(mock_other_widgets_1, mock_other_widgets_2, mock_other_widgets_3)
 
             widgets = Widget.find(:all, :include => [:singleton_widget, :other_widgets], :parent => @server)
 
-            widgets.first.singleton_widget.should == mock_singleton1
-            widgets.first.other_widgets.should == mock_other_widgets_1
-            widgets.last.singleton_widget.should == mock_singleton2
-            widgets.last.other_widgets.should == mock_other_widgets_2
+            widgets[0].singleton_widget.should == mock_singleton1
+            widgets[0].other_widgets.should == mock_other_widgets_1
+            widgets[1].singleton_widget.should == mock_singleton2
+            widgets[1].other_widgets.should == mock_other_widgets_2
+            widgets[2].singleton_widget.should == mock_singleton3
+            widgets[2].other_widgets.should == mock_other_widgets_3
           end
         end
       end
@@ -194,71 +296,83 @@ describe LucidWorks::Base do
         Widget.first(:parent => @server).should == mock_coll1
       end
     end
-  end
-
-  describe "instance methods" do
-
-    describe "collection_url" do
-      it "should generate a url from the server and its name" do
-        widget = Widget.new(:parent => @server, :id => 1234)
-        widget.collection_url.should == "#{@fake_server_uri}/widgets"
-      end
-    end
 
-    describe "member_url" do
-      it "should generate a url from the server, its name and its id" do
-        widget = Widget.new(:parent => @server, :id => 1234)
-        widget.member_url.should == "#{@fake_server_uri}/widgets/1234"
-      end
-    end
-
-    describe "attributes" do
-      before { @widget = Widget.new(:known_attr => 'known_value', :nil_attr => nil, :parent => @server) }
-
-      describe '#attr=' do
-        context "for an unknown attr" do
-          it "should set an attribute" do
-            @widget.new_attr = "fake_value"
-            @widget.new_attr.should == 'fake_value'
+    context "for a model with a schema" do
+      before :all do
+        class WidgetWithBigSchema < LucidWorks::Base
+          schema do
+            attribute :astring,      :string
+            attribute :large_number, :integer
+            attribute :iambool,      :boolean
+            #attribute :when,         :iso8601
           end
         end
-        context "for a known attr" do
-          it "should set an attribute" do
-            @widget.known_attr = "new_value"
-            @widget.known_attr.should == 'new_value'
+        class TrueClass
+          def to_yesno
+            self ? "yes" : "no"
           end
         end
       end
 
-      describe "#attr" do
-        context "for an unknown attr" do
-          it "should raise an error" do
-            lambda {
-              @widget.bogus_attr
-            }.should raise_error("Unknown attribute: 'bogus_attr'")
+      describe ".human_attribute_value(attribute, value)" do
+
+        context "for an attribute described as :boolean in the schema" do
+          it "should to_yesno the attribute" do
+            WidgetWithBigSchema.human_attribute_value(:iambool, true).should == "yes"
           end
         end
-        context "for a known attr" do
-          it "should return the attr" do
-            @widget.known_attr.should == 'known_value'
+
+        context "for an attribute described as :integer in the schema" do
+          before do
+            # So we can test without actually needing ActionPack
+            class LucidWorks::Base
+              def self.number_with_delimiter(x)
+                 x.to_s.gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
+              end
+            end
           end
-        end
-      end
 
-      describe "#attr?" do
-        context "for an unknown attr" do
-          it "should raise an error" do
-            lambda {
-              @widget.bogus_attr?
-            }.should raise_error("Unknown attribute: 'bogus_attr'")
+          it "should number_with_delimeter the attribute" do
+            WidgetWithBigSchema.human_attribute_value(:large_number, 12345678).should == "12,345,678"
           end
         end
-        context "for a known attr" do
-          it "should return true or false" do
-            @widget.known_attr?.should be_true
-            @widget.nil_attr?.should be_false
+
+        context "for an attributed described as :string in the schema" do
+          it "should just to_s the attribute" do
+            WidgetWithBigSchema.human_attribute_value(:astring, true).should == "true"
           end
         end
+
+        #context "for an attribute described as :iso8601 in the schema" do
+        #  it "should convert it to localtime to_s(:short)" do
+        #    @widget.human_attribute_value(:when).should == "19 Mar 10:40"
+        #  end
+        #end
+      end
+
+      describe ".to_select(attribute)" do
+        it "should create an array of arrays suitable for a form select" do
+          # Use Collection::Settings because we know de_duplication as translations
+          LucidWorks::Collection::Settings.to_select(:de_duplication).should ==
+            [['off', 'Off'], ['overwrite', 'Overwrite'], ['tag', 'Tag']]
+        end
+      end
+    end
+  end
+
+  describe "instance methods" do
+
+    describe "collection_url" do
+      it "should generate a url from the server and its name" do
+        widget = Widget.new(:parent => @server, :id => 1234)
+        widget.collection_url.should == "#{@fake_server_uri}/widgets"
+      end
+    end
+
+    describe "member_url" do
+      it "should generate a url from the server, its name and its id" do
+        widget = Widget.new(:parent => @server, :id => 1234)
+        widget.member_url.should == "#{@fake_server_uri}/widgets/1234"
       end
     end
 
@@ -266,7 +380,7 @@ describe LucidWorks::Base do
       context "for a new model" do
         context "with valid attributes" do
           before do
-            @widget_attrs = { :name => 'widget3', :size => 'large' }
+            @widget_attrs = { :name => 'widget4', :size => 'extra medium' }
             @widget = Widget.new @widget_attrs.merge(:parent => @server)
           end
 
@@ -360,6 +474,20 @@ describe LucidWorks::Base do
       end
     end
 
+    describe "#update_attributes" do
+      it "should update the model with the supplied attributes then call #save" do
+        widget_attrs = { :id => 123, :name => 'widget3', :size => 'large' }
+        widget = Widget.new widget_attrs.merge(:parent => @server)
+        widget.persisted = true
+        widget.should_receive(:save)
+
+        widget.update_attributes(:name => 'new_name', :size => 'new_size')
+
+        widget.name.should == 'new_name'
+        widget.size.should == 'new_size'
+      end
+    end
+
     describe "#destroy" do
       it "should call RestClient.delete with the appropriate URL" do
         widget = Widget.new(:parent => @server)
@@ -368,6 +496,12 @@ describe LucidWorks::Base do
         widget.destroy
       end
     end
+
+    describe "#human_attribute_value" do
+      it "should call class.human_attribute_value with attribute name and value" do
+        #TODO
+      end
+    end
   end
 
   describe "for a model with primary key other than 'id'" do

data/spec/lib/lucid_works/collection_spec.rb

@@ -126,8 +126,8 @@ describe LucidWorks::Collection do
       @collection.should be_valid
       @ds1 = LucidWorks::Datasource.create(
         :collection => @collection,
-        :crawler => "lucid.aperture",
-        :type => LucidWorks::Datasource::TYPES[:FileSystemDataSource][:type],
+        :crawler => LucidWorks::Datasource::CRAWLERS['file'],
+        :type => "file",
         :name => "US zoneinfo",
         :path => "/usr/share/zoneinfo/US",
         :crawl_depth => 1
@@ -135,8 +135,8 @@ describe LucidWorks::Collection do
       @ds1.should be_valid
       @ds2 = LucidWorks::Datasource.create(
         :collection => @collection,
-        :crawler => "lucid.aperture",
-        :type => LucidWorks::Datasource::TYPES[:WebDataSource][:type],
+        :crawler => LucidWorks::Datasource::CRAWLERS['web'],
+        :type => 'web',
         :name => "nothing_com",
         :url => "http://nothing.com/",
         :crawl_depth => 1
@@ -228,4 +228,16 @@ describe LucidWorks::Collection do
       @collection.empty!
     end
   end
+
+  describe "#fields" do
+    before do
+      @collection = @server.collections.first
+    end
+    it "should return a valid LucidWorks::Collection::Field" do
+      @collection.should respond_to(:field)
+      field = @collection.field('body')
+      field.should be_a(LucidWorks::Field)
+      field.should respond_to(:field_type)
+    end
+  end
 end