cloudsearchable 0.0.1

data/spec/cloudsearchable/cloudsearchable_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+require 'test_classes/cloud_searchable_test_class'
+
+describe Cloudsearchable do
+  let(:clazz){ CloudSearchableSampleClassFactory.call }
+
+  it 'can describe an index that returns ids for the class type' do
+    test_index = clazz.cloudsearch_index(:test_index)
+    test_index.should be_kind_of(Cloudsearchable::Domain)
+    test_index.fields.should have(4).items #3 explicit + 1 for the id of the object
+  end
+  
+  it 'has a default index' do
+    clazz.cloudsearch_index.should be_kind_of(Cloudsearchable::Domain)
+    clazz.cloudsearch_index(:test_index).should_not be(clazz.cloudsearch_index)
+  end
+  
+  it 'names domains consistent with CloudSearch limitations' do
+    clazz.cloudsearch_index(:test_index).name.should =~ /^[a-z][a-z0-9\-]+$/
+  end
+
+  describe 'an ordinary object' do
+    #An instance of the searchable class
+    let(:inst) do
+      inst = clazz.new
+      #arbitrary but plausible values for the core fields
+      inst.destroyed = false
+      inst.id = 42
+      inst.lock_version = 18
+      inst.customer = OpenStruct.new :id => 123
+      inst.name = "My Name"
+      inst
+    end
+
+    it 'supplies the right values to the fields' do
+      test_index = clazz.cloudsearch_index(:test_index)
+      test_index.fields[:test_class_id].value_for(inst).should be(inst.id)
+      test_index.fields[:customer_id].value_for(inst).should   be(inst.customer)
+      test_index.fields[:test_name].value_for(inst).should     be(inst.name)
+      test_index.fields[:helpfulness].value_for(inst).should   be(1234)
+    end
+    
+    it 'reindexes when told to' do
+      clazz.cloudsearch_index(           ).should_receive(:post_record).with(inst, inst.id, inst.lock_version)
+      clazz.cloudsearch_index(:test_index).should_receive(:post_record).with(inst, inst.id, inst.lock_version)
+      inst.update_indexes
+    end
+    
+    it 'generates a sensible addition sdf document' do
+      sdf = clazz.cloudsearch_index(:test_index).send :addition_sdf, inst, inst.id, inst.lock_version
+      sdf[:fields][:helpfulness].should be(1234)
+    end
+  end
+
+  describe 'a destroyed object' do
+    #An instance of the searchable class
+    let(:inst) do
+      inst = clazz.new
+      #arbitrary but plausible values for the core fields
+      inst.destroyed = true
+      inst
+    end
+
+    it 'reindexes when told to' do
+      clazz.cloudsearch_index(           ).should_receive(:delete_record).with(inst.id, inst.lock_version)
+      clazz.cloudsearch_index(:test_index).should_receive(:delete_record).with(inst.id, inst.lock_version)
+      inst.update_indexes
+    end
+  end
+
+end

data/spec/cloudsearchable/domain_spec.rb

@@ -0,0 +1,158 @@
+require 'spec_helper'
+require 'test_classes/cloud_searchable_test_class'
+
+describe Cloudsearchable::Domain do
+  before(:each) do
+    fake_client = double('client')
+    CloudSearch.stub(:client).and_return(fake_client)
+  end
+  
+  #
+  # First call to describe_domains returns it needs reindexing, 
+  # Second call returns that it is processing,
+  # Third call returns that it is done processing
+  #
+  let(:needs_rebuild_domain) do
+    described_class.new('nrb-index').tap do |dom|
+      resp = describe_domain_response(dom.name)
+      CloudSearch.client.stub(:describe_domains).with(:domain_names => [dom.name]).
+        and_return(
+          describe_domain_response(dom.name, :required_index_documents => true),
+          describe_domain_response(dom.name, :processing => true),
+          describe_domain_response(dom.name)
+        )
+    end
+  end
+  
+  # A domain name named 'my-index'
+  let(:domain) do
+    described_class.new('my-index').tap do |dom|
+      CloudSearch.client.stub(:describe_domains).and_return(describe_domain_response(dom.name))
+      CloudSearch.client.stub(:describe_domains).with(:domain_names => [dom.name]).and_return(describe_domain_response(dom.name))
+    end
+  end
+
+  let(:empty_domain) do
+    described_class.new('my-index').tap do |dom|
+      CloudSearch.client.stub(:describe_domains).and_return({})
+    end
+  end
+  
+  it 'can be instantiated' do
+    index = domain
+    index.name.should end_with('my-index')
+  end
+
+  it 'can haz a literal field' do
+    index = domain
+    index.add_field(:literary, :literal) { nil }
+    index.fields[:literary].type.should eq(:literal)
+  end
+
+  it 'can be initialized with a nested class' do
+    class OuterClassForCloudSearch
+      class InnerClass
+        include Cloudsearchable
+      end
+    end
+
+    OuterClassForCloudSearch::InnerClass.cloudsearch_prefix.should match(/^[A-Za-z0-9_-]+$/)
+    Object.instance_eval { remove_const :OuterClassForCloudSearch }
+  end
+  
+  context "SDF documents" do
+    let(:object) { OpenStruct.new(:field_with_nil_value => nil, :field_with_present_value => 42) }
+    subject do
+      described_class.new('my-index').tap do |d|
+        d.add_field(:field_with_nil_value, :literal)
+        d.add_field(:field_with_present_value, :literal)
+      end
+    end
+    
+    it "should generate present fields" do
+      subject.addition_sdf(object, "id", 1)[:fields][:field_with_present_value].should == 42
+    end
+    
+    it "should not generate nil fields" do
+      subject.addition_sdf(object, "id", 1)[:fields][:field_with_nil_value].should be_nil
+    end
+  end
+
+  it 'raises if the domain cannot be found' do
+    expect { empty_domain.send(:search_endpoint) }.to raise_error( Cloudsearchable::Domain::DomainNotFound,
+                                                                   /Cloudsearchable could not find the domain/)
+  end
+
+  #
+  # A test for the accidental clashing of domain caching
+  # on a class variable
+  #
+  it 'caches endpoints for multiple domains' do
+    domain.send(:search_endpoint).should_not eq(needs_rebuild_domain.send(:search_endpoint))
+  end
+
+  it 'endpoint selected is based on the domain name' do
+    domain.send(:search_endpoint).should eq describe_domain_response(domain.name)[:domain_status_list][0][:search_service][:endpoint]
+    domain.send(:doc_endpoint).should    eq describe_domain_response(domain.name)[:domain_status_list][0][:doc_service][:endpoint]
+  end
+  
+  it 'sleeps, waiting for reindexing' do
+    CloudSearch.client.should_receive(:index_documents).with(:domain_name => needs_rebuild_domain.name)
+    CloudSearch.client.should_receive(:describe_domains).exactly(3).times
+    needs_rebuild_domain.apply_changes(3).should == true
+  end
+
+  protected
+  
+  #
+  # A mockup of the response to a describe_domain request
+  #
+  def describe_domain_response(domain_name, options = {})
+    {
+      :domain_status_list=> [
+        {
+          :search_partition_count=>1, 
+          :search_service=>{
+            :arn=>"arn:aws:cs:us-east-1:510523556749:search/#{domain_name}", 
+            :endpoint=>"search-#{domain_name}-7bq6utq4fdrwax5r6irje7xlra.us-east-1.cloudsearch.amazonaws.com"
+          }, 
+          :num_searchable_docs=>23,
+          :search_instance_type=>"search.m1.small", 
+          :created=>true, 
+          :domain_id=>"510523556749/#{domain_name}", 
+          :processing=> options.fetch(:processing, false), 
+          :search_instance_count=>1, 
+          :domain_name=>"#{domain_name}", 
+          :requires_index_documents=> options.fetch(:required_index_documents,false), 
+          :deleted=>false, 
+          :doc_service=>{
+            :arn=>"arn:aws:cs:us-east-1:510523556749:doc/#{domain_name}", 
+            :endpoint=>"doc-#{domain_name}-7bq6utq4fdrwax5r6irje7xlra.us-east-1.cloudsearch.amazonaws.com"
+          }
+        },
+        {:search_partition_count=>1,
+         :search_service=>
+          {:arn=>
+            "arn:aws:cs:us-east-1:510523556749:search/dev-llarue-collection-items",
+           :endpoint=>
+            "search-dev-llarue-collection-items-hjopg2yzhcjdd4qxeglr2v5v7m.us-east-1.cloudsearch.amazonaws.com"},
+         :num_searchable_docs=>2,
+         :search_instance_type=>"search.m1.small",
+         :created=>true,
+         :domain_id=>"510523556749/dev-llarue-collection-items",
+         :processing=>false,
+         :search_instance_count=>1,
+         :domain_name=>"dev-llarue-collection-items",
+         :requires_index_documents=>false,
+         :deleted=>false,
+         :doc_service=>
+          {:arn=>"arn:aws:cs:us-east-1:510523556749:doc/dev-llarue-collection-items",
+           :endpoint=>
+            "doc-dev-llarue-collection-items-hjopg2yzhcjdd4qxeglr2v5v7m.us-east-1.cloudsearch.amazonaws.com"}}
+      ], 
+      :response_metadata=>{
+        :request_id=>"7d9487a7-1c9f-11e2-9f96-0958b8a97a74"
+      }
+    }
+  end
+end

data/spec/cloudsearchable/field_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+require 'ostruct'
+
+describe Cloudsearchable::Field do
+  it 'has a name' do
+    field = described_class.new 'fnord', :literal
+    field.name.should eq(:fnord)
+  end
+
+  it 'can find its value' do
+    test_value = nil
+    field = described_class.new('foo', :literal, :source => Proc.new { test_value })
+    test_value = 123
+    field.value_for(Object.new).should eq(test_value)
+
+    record = OpenStruct.new :a => test_value
+    field2 = described_class.new('bar', :literal, :source => :a)
+    field.value_for(record).should eq(test_value)
+  end
+
+  it 'generates a field definition' do
+    domain_name = 'narnia'
+    field = described_class.new('fnord', :literal, :search_enabled => true)
+    CloudSearch.client.should_receive(:define_index_field) do |call|
+      call[:domain_name].should eq(domain_name)
+      call[:index_field][:literal_options][:search_enabled].should be_true
+    end
+    field.define_in_domain domain_name
+  end
+end

data/spec/cloudsearchable/query_chain_spec.rb

@@ -0,0 +1,305 @@
+require 'spec_helper'
+require 'test_classes/cloud_searchable_test_class'
+
+describe Cloudsearchable::Query do
+  let(:clazz){ CloudSearchableSampleClassFactory.call }
+
+  it 'can build a simple search query' do
+    clazz.search.where(:customer_id, :eq, 12345).query.to_q[:bq].should =~ /customer_id:'12345'/
+  end
+
+  it 'chains' do
+    query = clazz.search.where(customer_id: 12345).where(helpfulness: 42).query.to_q[:bq]
+    query.should =~ /customer_id:'12345'/
+    query.should =~ /helpfulness:'42'/
+  end
+
+  it 'can build a query with "not equal to" condition' do
+    query = clazz.search.where(:customer_id, :!=, 1234).query.to_q[:bq].should =~ /\(not customer_id:'1234'\)/
+  end
+
+  it 'can build a query from a hash' do
+    query = clazz.search.where(customer_id: 12345, helpfulness: 42).query.to_q[:bq]
+    query.should =~ /customer_id:'12345'/
+    query.should =~ /helpfulness:'42'/
+  end
+
+  it "doesn't build queries without a query term" do
+    expect do
+      query = clazz.search.limit(10).query.to_q
+    end.to raise_exception
+  end
+
+  describe '#where' do
+    describe 'uint data type' do
+      describe 'within_range' do
+        it 'supports range query with uint fields' do
+          clazz.search.where(:helpfulness, :within_range, "0..#{123}").query.to_q[:bq].should =~ /helpfulness:0..123/
+        end
+
+        it 'supports range query with uint fields using a ruby range' do
+          clazz.search.where(:helpfulness, :within_range, 0..123).query.to_q[:bq].should =~ /helpfulness:0..123/
+        end
+      end
+
+      describe 'inequalities' do
+        it 'supports greater-than with uint fields' do
+          clazz.search.where(:helpfulness, :>, 123).query.to_q[:bq].should =~ /helpfulness:124../
+        end
+
+        it 'supports greater-than-or-equal-to with uint fields' do
+          clazz.search.where(:helpfulness, :>=, 123).query.to_q[:bq].should =~ /helpfulness:123../
+        end
+
+        it 'supports less-than with uint fields' do
+          clazz.search.where(:helpfulness, :<, 123).query.to_q[:bq].should =~ /helpfulness:..122/
+        end
+
+        it 'supports less-than-or-equal-to with uint fields' do
+          clazz.search.where(:helpfulness, :<=, 123).query.to_q[:bq].should =~ /helpfulness:..123/
+        end
+
+        it 'does not permit inequality operators with non-integer fields' do
+          [:>, :>=, :<, :<=].each do |op|
+            ['123', nil, 12.0].each do |value|
+              expect { clazz.search.where(:helpfulness, op, value).query.to_q[:bq] }.to raise_error
+            end
+          end
+        end
+      end
+    end
+  end
+
+
+  it 'supports querying for any of several values of a field' do
+    clazz.search.where(:category, :any, %w{big small}).query.to_q[:bq].should include("(or category:'big' category:'small')")
+  end
+
+  it 'supports text method' do
+    query = clazz.search.text('test').query.to_q[:q]
+    query.should =~ /test/
+  end
+
+  it 'supports chaining text and where clauses together' do
+    query = clazz.search.text('test').where(:featured, :==, 'f').query
+    query.to_q[:q].should  =~ /test/
+    query.to_q[:bq].should =~ /featured:'f'/
+  end
+
+  it 'supports ordering with a rank expression' do
+    clazz.search.where(customer_id: 12345).order('-helpfulness').query.to_q[:rank].should eq '-helpfulness'
+  end
+
+  it 'supports limit' do
+    clazz.search.where(customer_id: 12345).limit(10).query.to_q[:size].should eq 10
+  end
+
+  it 'has high default limit' do
+    clazz.search.where(customer_id: 12345).query.to_q[:size].should eq 100000
+  end
+
+  it 'supports offset' do
+    clazz.search.where(customer_id: 12345).offset(100).query.to_q[:start].should eq 100
+  end
+
+  context 'queries' do
+    before(:each) do
+      clazz.cloudsearch_index.should_receive(:execute_query).and_return(cloudsearch_response)
+    end
+
+    context 'query warning' do
+      before(:each) do
+        clazz.stub(:find).and_return([])
+        Cloudsearchable.logger.should_receive(:warn).with(/CS-InvalidFieldOrRankAliasInRankParameter/)
+      end
+
+      let(:query){clazz.search.where(customer_id: 12345).order("-adult")}
+      let(:cloudsearch_response) do
+        # generated by ranking a literal field that is search-enabled but not result-enabled
+        {
+            "rank" => "adult",
+            "match-expr" => "(label initialized_at:1363464074..1366056074)",
+            "hits" => {
+                "found" => 285,
+                "start" => 0,
+                "hit" => [
+                    {"id" => "40bdd5072b6dbae6245fe4ee837d22e3","data" => {"collection_item_id" => ["PCxSz65GIcZTtc0UpRdT-i--w-1365550370"]}},
+                    {"id" => "00af8f5f96aa1db7aff77be5651b3bb1","data" => {"collection_item_id" => ["PCxhksJTLRYnoGXvwZik82Fkw-1365020313"]}},
+                    {"id" => "00b6ac84e3ae402e7698959bf692a53e","data" => {"collection_item_id" => ["PCxs-fIVZnBcTzZ4MtfDguS1A-1365020274"]}},
+                    {"id" => "018fdee653bff74abd12ac30152a2837","data" => {"collection_item_id" => ["PCxmAGHFtAgyqUrgI3HgM_P6Q-1365548349"]}},
+                    {"id" => "01d062d24c389906eea2d16b8193eb56","data" => {"collection_item_id" => ["PCxqjaTmwydKM82NqymbryNfg-1365470479"]}},
+                    {"id" => "01e3ee5d848a30385a4e90eb851b094d","data" => {"collection_item_id" => ["PCxSz65GIcZTtc0UpRdT-i--w-1365550369"]}},
+                    {"id" => "01fca44cc596adb295ca6ee9f9f36499","data" => {"collection_item_id" => ["PCx7XKbKwOVf1VvEWvTl5c1Eg-1365020176"]}},
+                    {"id" => "02b85c9835b5045065ee389954a60c5f","data" => {"collection_item_id" => ["PCxp_xid_WeTfTmb5MySEfxhQ-1365115565"]}},
+                    {"id" => "040c01be434552a1d9e99eef9db87bdd","data" => {"collection_item_id" => ["PCxLOYzA4bCt7-bP6wsZnl-ow-1365020297"]}},
+                    {"id" => "048567c755e30d6d64d757508f1feaa0","data" => {"collection_item_id" => ["PCxJhhnpYkeSKrOxteQo5Jckw-1365115667"]}}
+                ]
+            },
+            "info" => {
+                "rid" => "7df344e77e1076a903e1f2dc1effcf3dde0a89442fb459d00a6e60ac64b8bbfcab1fbc5b35c10949",
+                "time-ms" => 3,
+                "cpu-time-ms" => 0,
+                "messages" => [
+                    {
+                        "severity" => "warning",
+                        "code" => "CS-InvalidFieldOrRankAliasInRankParameter",
+                        "host" => "7df344e77e1076a9884a6c43665da57c",
+                        "message" => "Unable to create score object for rank 'adult'"
+                    }
+                ]
+            }
+        }
+      end
+
+      it 'causes WarningInQueryResult exception' do
+        lambda{ query.to_a }.should raise_error(Cloudsearchable::WarningInQueryResult)
+      end
+
+      it 'takes a :fatal_warnings option, and when set to false, does not raise' do
+        sample_query = Cloudsearchable::QueryChain.new(double, fatal_warnings: false)
+        sample_query.instance_variable_get(:@fatal_warnings).should eq false
+
+        q = query
+        q.query.instance_variable_set(:@fatal_warnings, false)
+        lambda{ q.to_a }.should_not raise_error(Cloudsearchable::WarningInQueryResult)
+      end
+    end
+
+    context 'valid query results' do
+      let(:customer_id){ '12345' }
+      let(:other_customer_id){ 'foo' }
+
+      let(:cloudsearch_response) do
+        {
+            "rank"=>"-text_relevance",
+            "match-expr"=>"(label customer_id:'12345')",
+            "hits"=>{
+                "found"=>11,
+                "start"=>0,
+                "info"=>{
+                    "rid"=>"e2467862eecf73ec8dfcfe0cba1893abbe2e8803402f4da65b1195593c0f78ec3e8f1d29f6e40723",
+                    "time-ms"=>2,
+                    "cpu-time-ms"=>0
+                },
+                "hit"=>[
+                    {"id"=>"0633e1c9793f5288c58b664356533e81", "data"=>{"test_class_id"=>["ANINSTANCEID"]}},
+                # {"id"=>"04931ebede796ae8b435f1fd5291e772", "data"=>{"test_class_id"=>["PCxTj26ZRmV_EnHigQWx0S06w"]}},
+                # {"id"=>"72159a172d3043bfcdadb5244862b9ee", "data"=>{"test_class_id"=>["PCxS_apFtZMrKuqyPhFNstzMQ"]}},
+                # {"id"=>"1eb815b075bc005e97dc5827e53b9615", "data"=>{"test_class_id"=>["PCxSksjDUBehPWhYYW2Dtj4KQ"]}},
+                # {"id"=>"3e4950b829456b13bf1460b25a7aca26", "data"=>{"test_class_id"=>["PCx1oiyh6vrHGSeLvis4USMfQ"]}},
+                # {"id"=>"00b441f55fff86d2d746227988da77a9", "data"=>{"test_class_id"=>["PCxpt-aW8topsnTGs-AIkzWCA"]}},
+                # {"id"=>"919ea27d21bbdc07ead4688a0d7ceca1", "data"=>{"test_class_id"=>["PCxFHGLbGJ2mzau_a6-gh5ORw"]}},
+                # {"id"=>"c663c2d9af342b0038fc808322143cfd", "data"=>{"test_class_id"=>["PCxyFwShwjWBp_WiXB0rFb2WA"]}},
+                # {"id"=>"de8f00af5636393e2553c4b4710d3393", "data"=>{"test_class_id"=>["PCxnqXfm8McflBgi4HsYoUXVw"]}},
+                # {"id"=>"e297cf21741a4c43697ea2586164a987", "data"=>{"test_class_id"=>["PCxrdk8gAEVbkuUCazu2-qLjQ"]}}
+                ]
+            }
+        }
+      end
+
+      it 'materializes' do
+        clazz.should_receive(:find).with(["ANINSTANCEID"]).and_return([customer_id])
+        query = clazz.search.where(customer_id: 12345)
+        query.to_a.should == [customer_id]
+      end
+
+      it 'materializes db results only once' do
+        expected_results = [customer_id, other_customer_id]
+        clazz.should_receive(:find).once.and_return(expected_results)
+
+        query = clazz.search.where(customer_id: 12345)
+        query.materialize!
+        query.materialize!
+      end
+
+      it 'should not materialize if only asking for found_count' do
+        clazz.should_not_receive(:find)
+        clazz.search.where(customer_id: 12345).found_count
+      end
+
+      it 'supports each for multiple results' do
+        expected_results = [customer_id, other_customer_id]
+        clazz.should_receive(:find).with(["ANINSTANCEID"]).and_return(expected_results)
+
+        results = clazz.search.where(customer_id: 12345).to_a
+        (0..results.length).each{ |i| results[i].should eq expected_results[i] }
+      end
+
+      it 'supports each for single results' do
+        clazz.should_receive(:find).with(["ANINSTANCEID"]).and_return(customer_id)
+
+        results = clazz.search.where(customer_id: 12345).to_a
+        results.each{ |r| r.should eq customer_id }
+      end
+
+      it 'supports each for nil result' do
+        clazz.should_receive(:find).with(["ANINSTANCEID"]).and_return(nil)
+
+        results = clazz.search.where(customer_id: 12345).to_a
+        results.each{ |r| r.should_not be }
+      end
+
+      it 'uses materialized method' do
+        clazz.should_receive(:another_find).with(["ANINSTANCEID"]).and_return(customer_id)
+        clazz.materialize_method :another_find
+        clazz.search.where(customer_id: 12345).to_a
+      end
+
+      it 'returns the correct found count' do
+        clazz.search.where(customer_id: 12345).found_count.should == 11
+      end
+    end
+
+    context 'invalid query results' do
+      let(:cloudsearch_response) do
+        {
+            "error"=>"info",
+            "rid"=>"6ddcaa561c05c4cc85ddb10cb46568af2ef64b0583910e32210f551c238586e40fc3abe629ca87b250796d395a628af6",
+            "time-ms"=>20,
+            "cpu-time-ms"=>0,
+            "messages"=>[
+                {
+                    "severity"=>"fatal",
+                    "code"=>"CS-UnknownFieldInMatchExpression",
+                    "message"=>"Field 'asdf' is not defined in the metadata for this collection."
+                }
+            ]
+        }
+      end
+
+      it 'raises an exception when requesting found count with an error response' do
+        expect { clazz.search.where(customer_id: 12345).found_count }.to raise_error
+      end
+    end
+
+    context 'empty results with non-empty data' do
+      let(:cloudsearch_response) do
+        {
+            # Empty-yet-present data may occur with a NOT query, such as "(not customer_id:'XYZ')".
+            # Refer to: https://aws.amazon.com/support/case?caseId=107084141&language=en
+            "rank" => "-text_relevance",
+            "match-expr" => "(not customer_id:'A3E4T85Q6WPY4F')",
+            "hits" => {
+                "found" => 2087,
+                "start" => 0,
+                "hit" => [
+                    {"id" => "fb9fb53e32c4b3714cf39be4b855d34b", "data" => { "test_class_id" => []}},
+                ]
+            },
+            "info" => {
+                "rid" => "621cf310b88f32076b1908e45b4930aafb872497bdbf3b5e64065619c0dcec96bbe513281093d6c7",
+                "time-ms" => 3,
+                "cpu-time-ms" => 0
+            }
+        }
+      end
+
+      it 'does not raise an exception' do
+        clazz.should_receive(:find).with([]).and_return(nil)
+        clazz.search.where(:customer_id, :!=, 'ABCDE')
+        expect { clazz.search.where(:customer_id, :!=, 'ABCDE').to_a }.to_not raise_error
+      end
+    end
+  end
+
+end