cloudsearchable 0.0.1

data/lib/cloudsearchable/domain.rb

@@ -0,0 +1,159 @@
+#
+# Class the represents the schema of a domain in CloudSearch
+#
+# In general, it will be instantiated by a helper class or module, like Cloudsearch_enabled.
+#
+module Cloudsearchable
+  class Domain
+    class DomainNotFound < StandardError; end
+
+    attr_reader :name, :fields
+
+    def initialize name
+      @name = "#{Cloudsearchable::Config.domain_prefix}#{name}"
+      @fields = {}
+    end
+
+    # Defines a literal index field.
+    # @param name field name
+    # @param type field type - one of :literal, :uint, or :text
+    # @option options [Boolean] :search_enabled (true)
+    # @option options [Boolean] :return_enabled (true)
+    # @option options [Symbol or Proc] :source The name of a method to call on a record to fetch
+    #   the value of the field, or else a Proc to be evaluated in the context of the record.
+    #   Defaults to a method with the same name as the field.
+    def add_field(name, type, options = {})
+      field = Field.new(name, type, options)
+      raise "Field #{name} already exists on index #{self.name}" if @fields.has_key?(field.name)
+      @fields[field.name] = field
+    end
+
+    # Creates the domain and defines its index fields in Cloudsearch
+    # Will blindly recreate index fields, no-op if the index already exists
+    def create
+      Cloudsearchable.logger.info "Creating domain #{name}"
+      CloudSearch.client.create_domain(:domain_name => name)
+    
+      #Create the fields for the index
+      fields.values.each do |field| 
+        Cloudsearchable.logger.info "  ...creating #{field.type} field #{name}"
+        field.define_in_domain self.name
+      end
+      Cloudsearchable.logger.info "  ...done!"
+    end
+
+    def reindex
+      CloudSearch.client.index_documents(:domain_name => name)
+    end
+    
+    #
+    # This queries the status of the domain from Cloudsearch and determines if 
+    # the domain needs to be reindexed. If so, it will initiate the reindex and
+    # wait timeout seconds for it to complete. Default is 0. Reindexings tend
+    # to take 15-30 minutes. 
+    #
+    # @return true if the changes are applied, false if the domain is still reindexing
+    #
+    def apply_changes(timeout = 0)
+      d = cloudsearch_domain(true)[:domain_status_list][0]
+      if(d[:requires_index_documents])
+        reindex
+      end
+      
+      #We'll potentially sleep until the reindex has completed
+      end_time = Time.now + timeout
+      sleep_time = 1
+      loop do
+        d = cloudsearch_domain(true)[:domain_status_list][0]
+        break unless (d[:processing] && Time.now < end_time)
+        
+        sleep(sleep_time)          
+        sleep_time = [2 * sleep_time, end_time - Time.now].min #exponential backoff
+      end
+      
+      !d[:processing] #processing is true as long as it is reindexing
+    end
+
+    # Add or replace the CloudSearch document for a particular version of a record
+    def post_record record, record_id, version
+      CloudSearch.post_sdf doc_endpoint, addition_sdf(record, record_id, version)
+    end
+
+    # Delete the CloudSearch document for a particular record (version must be greater than the last version pushed)
+    def delete_record record_id, version
+      CloudSearch.post_sdf doc_endpoint, deletion_sdf(record_id, version)
+    end
+
+    def execute_query(params)
+      uri    = URI("http://#{search_endpoint}/#{CloudSearch::API_VERSION}/search")
+      uri.query = URI.encode_www_form(params)
+      Cloudsearchable.logger.info "CloudSearch execute: #{uri.to_s}"
+      res = Net::HTTP.get_response(uri).body
+      JSON.parse(res)
+    end
+
+    def deletion_sdf record_id, version
+      {
+        :type    => "delete",
+        :id      => document_id(record_id),
+        :version => version
+      }
+    end
+
+    def addition_sdf record, record_id, version
+      {
+        :type    => "add",
+        :id      => document_id(record_id),
+        :version => version,
+        :lang    => "en", # FIXME - key off of marketplace
+        :fields  => sdf_fields(record)
+      }
+    end
+
+    # Generate a documentID that follows the CS restrictions
+    def document_id record_id
+      Digest::MD5.hexdigest record_id.to_s
+    end
+
+    protected
+
+    #
+    # AWS Cloudsearchable Domain
+    #
+    # @param force_reload force a re-fetch from the domain
+    #
+    def cloudsearch_domain(force_reload = false)
+      if(force_reload || !@domain)
+        @domain = CloudSearch.client.describe_domains(:domain_names => [name])
+      else
+        @domain
+      end
+
+      status = @domain[:domain_status_list]
+      if status.nil? || status && status.empty?
+        raise(DomainNotFound, "Cloudsearchable could not find the domain '#{name}' in AWS. Check the name and the availability region.")
+      end
+
+      @domain
+    end
+
+    def sdf_fields record
+      fields.values.inject({}) do |sdf, field|
+        value = field.value_for(record)
+        sdf[field.name] = value if value
+        sdf
+      end
+    end
+
+    # AWS CloudSearch Domain API to get search endpoint
+    def search_endpoint
+      @search_endpoint ||= cloudsearch_domain[:domain_status_list].first[:search_service][:endpoint]
+    end
+
+    # AWS CloudSearch Domain API to get doc endpoint
+    def doc_endpoint
+      @doc_endpoint ||= cloudsearch_domain[:domain_status_list].first[:doc_service][:endpoint]
+    end
+
+  end
+end

data/lib/cloudsearchable/field.rb

@@ -0,0 +1,56 @@
+require 'active_support/core_ext/hash'
+
+#
+# Class the represents the schema of a domain in CloudSearch
+#
+# In general, it will be instantiated by a helper class or module, like Cloudsearch_enabled.
+#
+module Cloudsearchable
+  # Represents a single field in a CloudSearch index.
+  #
+  class Field
+    FieldTypes = [:literal, :uint, :text].freeze
+    # Maps the type of field to the name of the options hash when defining the field
+    FieldTypeOptionsNames = {:literal => :literal_options, :uint => :u_int_options, :text => :text_options}.freeze
+    # Maps from field type to the allowed set of options for the field
+    FieldTypeOptionsKeys = {
+      literal: [:default_value, :facet_enabled, :search_enabled, :result_enabled].freeze,
+      uint:    [:default_value].freeze,
+      text:    [:default_value, :facet_enabled, :result_enabled].freeze
+    }.freeze
+    attr_reader :name, :type, :source, :options
+
+    def initialize(name, type, options = {})
+      raise ArgumentError, "Invalid field type '#{type}'" unless FieldTypes.include?(type)
+      @name = name.to_sym
+      @type = type.to_sym
+      @source = options[:source] || @name
+      @options = options.slice(*FieldTypeOptionsKeys[@type])
+    end
+
+    def value_for record
+      if @source.respond_to?(:call)
+        record.instance_exec &@source
+      else
+        record.send @source
+      end
+    end
+
+    def define_in_domain domain_name
+      CloudSearch.client.define_index_field(
+        :domain_name => domain_name,
+        :index_field => definition
+      )
+    end
+
+    def definition
+      # http://docs.amazonwebservices.com/cloudsearch/latest/developerguide/API_IndexField.html
+      {
+        :index_field_name => name.to_s, 
+        :index_field_type => type.to_s,
+        FieldTypeOptionsNames[type] => options
+      }
+    end
+    protected :definition
+  end
+end

data/lib/cloudsearchable/query_chain.rb

@@ -0,0 +1,218 @@
+module Cloudsearchable
+  class NoClausesError < StandardError; end
+  class WarningInQueryResult < StandardError; end
+
+  #
+  # An object that represents a query to cloud search
+  #
+  class QueryChain
+    include Enumerable
+    
+    attr_reader :domain, :fields
+
+    # options:
+    #   - fatal_warnings: if true, raises a WarningInQueryResult exception on warning. Defaults to false
+    def initialize(domain, options = {})
+      @fatal_warnings = options.fetch(:fatal_warnings, false)
+      @domain         = domain
+      @q              = nil
+      @clauses        = []
+      @rank           = nil
+      @limit          = 100000 # 10 is the CloudSearch default, 2kb limit will probably hit before this will
+      @offset         = nil
+      @fields         = Set.new
+      @results        = nil
+    end
+    
+    #
+    # This method can be called in several different forms. 
+    #
+    # To do an equality search on several fields, you can pass a single hash, e.g.:
+    #
+    #   Collection.search.where(customer_id: "12345", another_field: "Some value")
+    #
+    # To do a search on a single field, you can pass three parameters in the 
+    # form: where(field, op, value)
+    #
+    #   Collection.search.where(:customer_id, :==, 12345)
+    #
+    # To search for any of several possible values for a field, use the :any operator:
+    #
+    #   Collection.search.where(:product_group, :any, %w{gl_kitchen gl_grocery})
+    #
+    # Equality and inequality operators (:==, :!=, :<, :<=, :>, :>=) are supported on
+    # integers, and equality operators are supported on all scalars.
+    # Currently, special operators against arrays (any and all) are not yet implemented.
+    #
+    def where(field_or_hash, op = nil, value = nil)
+      raise if materialized?
+
+      if field_or_hash.is_a? Hash
+        field_or_hash.each_pair do |k, v| 
+          where(k, :==, v)
+        end
+      elsif field_or_hash.is_a? Symbol
+        field = field_or_hash
+        @clauses << if op == :within_range
+                      "#{field}:#{value.to_s}"
+                    elsif op == :== || op == :eq
+                      "#{field}:'#{value.to_s}'"
+                    elsif op == :any
+                      '(or ' + value.map { |v| "#{field}:'#{v.to_s}'" }.join(' ') + ')'
+                    elsif op == :!=
+                      "(not #{field}:'#{value.to_s}')"
+                    elsif op == :> && value.is_a?(Integer)
+                      "#{field}:#{value+1}.."
+                    elsif op == :< && value.is_a?(Integer)
+                      "#{field}:..#{value-1}"
+                    elsif op == :>= && value.is_a?(Integer)
+                      "#{field}:#{value}.."
+                    elsif op == :<= && value.is_a?(Integer)
+                      "#{field}:..#{value}"
+                    else
+                      raise "op #{op} is unrecognized"
+                    end
+      else
+        raise "field_or_hash must be a Hash or Symbol, not a #{field_or_hash.class}"
+      end
+
+      self
+    end
+
+    #
+    # Allows searching by text, overwriting any existing text search.
+    #
+    #   Collection.search.text('mens shoes')
+    #
+    # For more examples see http://docs.aws.amazon.com/cloudsearch/latest/developerguide/searching.text.html
+    #
+    def text(text)
+      raise if materialized?
+      @q = text
+      self
+    end
+
+    #
+    # Set a rank expression on the query, overwriting any existing expression. Defaults to "-text_relevance"
+    #
+    #   Collection.search.order('created_at')  # order by the created_at field ascending
+    #   Collection.search.order('-created_at') # descending order
+    #
+    # For more examples see http://docs.amazonwebservices.com/cloudsearch/latest/developerguide/tuneranking.html
+    #
+    def order rank_expression
+      raise if materialized?
+      raise "order clause must be a string, not a #{rank_expression.class}" unless rank_expression.is_a? String
+      @rank = rank_expression.to_s
+      self
+    end
+
+    #
+    # Limit the number of results returned from query to the given count.
+    #
+    #   Collection.search.limit(25)
+    #
+    def limit count
+      raise if materialized?
+      raise "limit value must be must respond to to_i, #{count.class} does not" unless count.respond_to? :to_i
+      @limit = count.to_i
+      self
+    end
+
+    #
+    # Offset the results returned by the query by the given count.
+    #
+    #   Collection.search.offset(250)
+    #
+    def offset count
+      raise if materialized?
+      raise "limit value must be must respond to to_i, #{count.class} does not" unless count.respond_to? :to_i
+      @offset = count.to_i
+      self
+    end
+
+    #
+    # Adds a one or more fields to the returned result set, e.g.:
+    #   
+    #   my_query.returning(:collection_id)
+    #   my_query.returning(:collection_id, :created_at)
+    #
+    #   x = [:collection_id, :created_at]
+    #   my_query.returning(x)
+    #
+    def returning(*fields)
+      raise if materialized?
+
+      fields.flatten!
+      fields.each do |f|
+        @fields << f
+      end
+      self
+    end
+    
+    #
+    # True if the query has been materialized (e.g. the search has been 
+    # executed). 
+    #
+    def materialized?
+      !@results.nil?
+    end
+    
+    #
+    # Executes the query, getting a result set, returns true if work was done,
+    # false if the query was already materialized.
+    # Raises exception if there was a warning and not in production.
+    #
+    def materialize!
+      return false if materialized?
+      
+      @results = domain.execute_query(to_q)
+
+      if @results && @results["info"] && messages = @results["info"]["messages"]
+        messages.each do |message|
+          if message["severity"] == "warning"
+            Cloudsearchable.logger.warn "Cloud Search Warning: #{message["code"]}: #{message["message"]}"
+            raise(WarningInQueryResult, "#{message["code"]}: #{message["message"]}") if @fatal_warnings
+          end
+        end
+      end
+
+      true
+    end
+
+    def found_count
+      materialize!
+      if @results['hits']
+        @results['hits']['found']
+      else
+        raise "improperly formed response. hits parameter not available. messages: #{@results["messages"]}"
+      end
+    end
+    
+    def each(&block)
+      materialize!
+     if @results['hits']
+       @results['hits']['hit'].each(&block)
+     else
+       raise "improperly formed response. hits parameter not available. messages: #{@results["messages"]}"
+     end
+    end
+    
+    #
+    # Turns this Query object into a query string hash that goes on the CloudSearch URL
+    #
+    def to_q
+      raise NoClausesError, "no search terms were specified" if (@clauses.nil? || @clauses.empty?) && (@q.nil? || @q.empty?)
+      
+      bq = (@clauses.count > 0) ? "(and #{@clauses.join(' ')})" : @clauses.first
+      {
+        q: @q,
+        bq: bq,
+        rank: @rank,
+        size: @limit,
+        start: @offset,
+        :'return-fields' => @fields.reduce("") { |s,f| s << f.to_s }
+      }
+    end
+  end
+end

data/lib/cloudsearchable/version.rb

@@ -0,0 +1,3 @@
+module Cloudsearchable
+  VERSION = "0.0.1"
+end

data/spec/cloudsearchable/cloud_search_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+require 'test_classes/cloud_searchable_test_class'
+
+describe CloudSearch do
+
+  let(:item) do
+    CloudSearchableSampleClassFactory.call.new.tap do |instance|
+      instance.destroyed = false
+      instance.lock_version = 1
+      instance.id = 1
+      instance.customer = '1234'
+    end
+  end
+  let(:sdf_document){item.class.cloudsearch_index(:test_index).send :addition_sdf, item, item.id, item.lock_version}
+  let(:endpoint){'https://fake_end_point.amazon.com'}
+
+  class MockHTTPOK < Net::HTTPOK
+    attr :body
+    def initialize body
+      @body = body
+    end
+  end
+
+  class MockHTTPBadRequest < Net::HTTPBadRequest
+    def initialize; end
+  end
+
+  let(:success_response){ MockHTTPOK.new( {"status" => "success", "adds" => 1, "deletes" => 0}.to_json ) }
+
+  it 'json parses the response' do
+    Net::HTTP.any_instance.stub(:start).and_return{success_response}
+
+    response = described_class.post_sdf endpoint, sdf_document
+    response.should eq JSON.parse success_response.body
+  end
+
+  it 'triggers error! on response its no not a Net::HTTPSuccess' do
+    response = MockHTTPBadRequest.new
+    Net::HTTP.any_instance.stub(:start).and_return{response}
+
+    response.should_receive(:error!)
+    described_class.post_sdf endpoint, sdf_document
+  end
+
+end