elastictastic 0.5.0

data/lib/elastictastic/persistence.rb

@@ -0,0 +1,67 @@
+module Elastictastic
+  module Persistence
+    def save
+      persisted? ? update : create
+    end
+    
+    def destroy
+      if persisted?
+        Elastictastic.persister.destroy(self)
+      else
+        raise OperationNotAllowed, "Cannot destroy transient document: #{inspect}"
+      end
+    end
+
+    def persisted?
+      !!@persisted
+    end
+
+    def transient?
+      !persisted?
+    end
+
+    def pending_save?
+      !!@pending_save
+    end
+
+    def pending_destroy?
+      !!@pending_destroy
+    end
+
+    def persisted!
+      @persisted = true
+      @pending_save = false
+    end
+
+    def transient!
+      @persisted = @pending_destroy = false
+    end
+
+    def pending_save!
+      @pending_save = true
+    end
+
+    def pending_destroy!
+      @pending_destroy = true
+    end
+
+    protected
+
+    def create
+      Elastictastic.persister.create(self)
+    end
+
+    def update
+      Elastictastic.persister.update(self)
+    end
+
+    private
+
+    def assert_transient!
+      if persisted?
+        raise IllegalModificationError,
+          "Cannot modify identity attribute after model has been saved."
+      end
+    end
+  end
+end

data/lib/elastictastic/properties.rb

@@ -0,0 +1,236 @@
+module Elastictastic
+  module Properties
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      def each_field
+        properties.each_pair do |field, properties|
+          if properties['properties']
+            embeds[field].clazz.each_field do |embed_field, embed_properties|
+              yield("#{field}.#{embed_field}", embed_properties)
+            end
+          elsif properties['fields']
+            properties['fields'].each_pair do |variant_field, variant_properties|
+              if variant_field == field
+                yield(field, variant_properties)
+              else
+                yield("#{field}.#{variant_field}", variant_properties)
+              end
+            end
+          else
+            yield field, properties
+          end
+        end
+      end
+
+      def select_fields
+        [].tap do |fields|
+          each_field do |field, properties|
+            fields << [field, properties] if yield(field, properties)
+          end
+        end
+      end
+
+      def all_fields
+        @all_fields ||= {}.tap do |fields|
+          each_field { |field, properties| fields[field] = properties }
+        end
+      end
+
+      def field_properties
+        @field_properties ||= {}
+      end
+
+      def properties
+        return @properties if defined? @properties
+        @properties = {}
+        @properties.merge!(field_properties)
+        embeds.each_pair do |name, embed|
+          @properties[name] = { 'properties' => embed.clazz.properties }
+        end
+        @properties
+      end
+
+      def properties_for_field(field_name)
+        properties[field_name.to_s]
+      end
+
+      def embeds
+        @embeds ||= {}
+      end
+
+      def field(*field_names, &block)
+        options = field_names.extract_options!
+        field_names.each do |field_name|
+          define_field(field_name, options, &block)
+        end
+      end
+
+      def define_field(field_name, options, &block)
+        field_name = field_name.to_s
+
+        module_eval(<<-RUBY, __FILE__, __LINE__ + 1)
+          def #{field_name}
+            read_attribute(#{field_name.inspect})
+          end
+
+          def #{field_name}=(value)
+            write_attribute(#{field_name.inspect}, value)
+          end
+        RUBY
+
+        field_properties[field_name.to_s] =
+          Field.process(field_name, options, &block)
+      end
+
+      def embed(*embed_names)
+        options = embed_names.extract_options!
+
+        embed_names.each do |embed_name|
+          define_embed(embed_name, options)
+        end
+      end
+
+      def define_embed(embed_name, options)
+        embed_name = embed_name.to_s
+        embed = Association.new(embed_name, options)
+
+        module_eval <<-RUBY, __FILE__, __LINE__ + 1
+          def #{embed_name}
+            read_embed(#{embed_name.inspect})
+          end
+
+          def #{embed_name}=(value)
+            Util.call_or_each(value) do |check_value|
+              unless check_value.nil? || check_value.is_a?(#{embed.class_name})
+                raise TypeError, "Expected instance of class #{embed.class_name}; got \#{check_value.inspect}"
+              end
+            end
+            write_embed(#{embed_name.inspect}, value)
+          end
+        RUBY
+
+        embeds[embed_name] = embed
+      end
+    end
+
+    module InstanceMethods
+      def initialize(attributes = {})
+        super
+        @attributes = {}
+        @embeds = {}
+        self.attributes = attributes
+      end
+
+      def attributes
+        @attributes.with_indifferent_access
+      end
+
+      def attributes=(attributes)
+        attributes.each_pair do |field, value|
+          __send__(:"#{field}=", value)
+        end
+      end
+
+      def elasticsearch_doc
+        {}.tap do |doc|
+          @attributes.each_pair do |field, value|
+            next if value.nil?
+            doc[field] = Util.call_or_map(value) do |item|
+              serialize_value(field, item)
+            end
+          end
+          @embeds.each_pair do |field, embedded|
+            next if embedded.nil?
+            doc[field] = Util.call_or_map(embedded) do |item|
+              item.elasticsearch_doc
+            end
+          end
+        end
+      end
+
+      def elasticsearch_doc=(doc)
+        return if doc.nil?
+        doc.each_pair do |field_name, value|
+          if self.class.properties.has_key?(field_name)
+            embed = self.class.embeds[field_name]
+            if embed
+              embedded = Util.call_or_map(value) do |item|
+                embed.clazz.new.tap { |e| e.elasticsearch_doc = item }
+              end
+              write_embed(field_name, embedded)
+            else
+              deserialized = Util.call_or_map(value) do |item|
+                deserialize_value(field_name, item)
+              end
+              write_attribute(field_name, deserialized)
+            end
+          end
+        end
+      end
+
+      protected
+
+      def read_attribute(field)
+        @attributes[field.to_s]
+      end
+
+      def write_attribute(field, value)
+        if value.nil?
+          @attributes.delete(field.to_s)
+        else
+          @attributes[field.to_s] = value
+        end
+      end
+
+      def read_attributes
+        @attributes
+      end
+
+      def write_attributes(attributes)
+        @attributes = attributes
+      end
+
+      def read_embed(field)
+        @embeds[field.to_s]
+      end
+
+      def write_embed(field, value)
+        @embeds[field.to_s] = value
+      end
+
+      private
+
+      def serialize_value(field_name, value)
+        type = self.class.properties_for_field(field_name)['type'].to_s
+        case type
+        when 'date'
+          time = value.to_time
+          time.to_i * 1000 + time.usec / 1000
+        when 'integer', 'byte', 'short', 'long'
+          value.to_i
+        when 'float', 'double'
+          value.to_f
+        when 'boolean'
+          !!value
+        else
+          value
+        end
+      end
+
+      def deserialize_value(field_name, value)
+        return nil if value.nil?
+        if self.class.properties_for_field(field_name)['type'].to_s == 'date'
+          if value.is_a? Fixnum
+            sec, usec = value / 1000, (value % 1000) * 1000
+            Time.at(sec, usec).utc
+          else
+            Time.parse(value)
+          end
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/elastictastic/railtie.rb

@@ -0,0 +1,35 @@
+module Elastictastic
+  class Railtie < Rails::Railtie
+    config.elastictastic = Elastictastic.config
+
+    initializer "elastictastic.configure_rails" do
+      config_path = Rails.root.join('config/elastictastic.yml').to_s
+      config = Elastictastic.config
+      app_name = Rails.application.class.name.split('::').first.underscore
+      config.default_index = "#{app_name}_#{Rails.env}"
+
+      if File.exist?(config_path)
+        yaml = YAML.load_file(config_path)[Rails.env]
+        if yaml
+          yaml.each_pair do |name, value|
+            config.__send__("#{name}=", value)
+          end
+        end
+      end
+
+      Elastictastic.config.logger = Rails.logger
+
+      require 'elastictastic/new_relic_instrumentation' if defined? NewRelic
+    end
+
+    initializer "elastictastic.instantiate_observers" do
+      config.after_initialize do
+        ::Elastictastic::Observing.instantiate_observers
+
+        ActionDispatch::Callbacks.to_prepare do
+          ::Elastictastic.instantiate_observers
+        end
+      end
+    end
+  end
+end

data/lib/elastictastic/resource.rb

@@ -0,0 +1,4 @@
+warn 'Elastictastic::Resource is deprecated and will be removed in a future version. The new module name is Elastictastic::NestedDocument'
+module Elastictastic
+  Resource = NestedDocument
+end

data/lib/elastictastic/scope.rb

@@ -0,0 +1,283 @@
+require 'hashie'
+
+module Elastictastic
+  class Scope < BasicObject
+    attr_reader :clazz, :index
+
+    def initialize(index, clazz, search = Search.new, parent_collection = nil)
+      @index, @clazz, @search, @parent_collection =
+        index, clazz, search, parent_collection
+    end
+
+    def initialize_instance(instance)
+      index = @index
+      instance.instance_eval { @index = index }
+    end
+
+    def params
+      @search.params
+    end
+
+    def each
+      if ::Kernel.block_given?
+        find_each { |result, hit| yield result }
+      else
+        ::Enumerator.new(self, :each)
+      end
+    end
+
+    def find_each(batch_options = {}, &block)
+      if block
+        find_in_batches(batch_options) { |batch| batch.each(&block) }
+      else
+        ::Enumerator.new(self, :find_each, batch_options)
+      end
+    end
+
+    def find_in_batches(batch_options = {}, &block)
+      return ::Enumerator.new(self, :find_in_batches, batch_options) unless block
+      if params.key?('size') || params.key?('from')
+        yield search_all
+      elsif params.key?('sort') || params.key('facets')
+        search_in_batches(&block)
+      else
+        scan_in_batches(batch_options, &block)
+      end
+    end
+
+    def count
+      return @count if defined? @count
+      populate_counts
+      @count
+    end
+
+    def empty?
+      count == 0
+    end
+
+    def any?(&block)
+      block ? each.any?(&block) : !empty?
+    end
+
+    def first
+      params = from(0).size(1).params
+      hit = ::Elastictastic.client.search(
+        @index,
+        @clazz.type,
+        params
+      )['hits']['hits'].first
+      materialize_hit(hit) if hit
+    end
+
+    def all
+      scoped({})
+    end
+
+    def all_facets
+      return @all_facets if defined? @all_facets
+      populate_counts
+      @all_facets ||= nil
+    end
+
+    def scoped(params, index = @index)
+      ::Elastictastic::Scope.new(
+        @index,
+        @clazz,
+        @search.merge(Search.new(params)),
+        @parent_collection
+      )
+    end
+
+    def destroy_all
+      #FIXME support delete-by-query
+      ::Elastictastic.client.delete(@index, @clazz.type)
+    end
+
+    def sync_mapping
+      #XXX is this a weird place to have this?
+      ::Elastictastic.client.put_mapping(index, type, @clazz.mapping)
+    end
+
+    def find(*ids)
+      #TODO support combining this with other filters/query
+      force_array = ::Array === ids.first
+      ids = ids.flatten
+      if ::Hash === ids.first
+        find_many_in_many_indices(*ids)
+      elsif ids.length == 1
+        instance = find_one(ids.first)
+        force_array ? [instance] : instance
+      else
+        find_many(ids)
+      end
+    end
+
+    Search::KEYS.each do |search_key|
+      module_eval <<-RUBY
+        def #{search_key}(*values, &block)
+          values << ScopeBuilder.build(&block) if block
+
+          case values.length
+          when 0 then ::Kernel.raise ::ArgumentError, "wrong number of arguments (0 for 1)"
+          when 1 then value = values.first
+          else value = values
+          end
+
+          scoped(#{search_key.inspect} => value)
+        end
+      RUBY
+    end
+
+    def method_missing(method, *args, &block)
+      if ::Enumerable.method_defined?(method)
+        each.__send__(method, *args, &block)
+      elsif @clazz.respond_to?(method)
+        @clazz.with_scope(self) do
+          @clazz.__send__(method, *args, &block)
+        end
+      else
+        super
+      end
+    end
+
+    def inspect
+      inspected = "#{@clazz.name}:#{@index.name}"
+      inspected << @search.params.to_json unless @search.params.empty?
+      inspected
+    end
+
+    protected
+
+    def search(search_params = {})
+      ::Elastictastic.client.search(
+        @index,
+        @clazz.type,
+        params,
+        search_params
+      )
+    end
+
+    private
+
+    def search_all
+      response = search(:search_type => 'query_then_fetch')
+      populate_counts(response)
+      materialize_hits(response['hits']['hits'])
+    end
+
+    def search_in_batches(&block)
+      from, size = 0, ::Elastictastic.config.default_batch_size
+      scope_with_size = self.size(size)
+      begin
+        scope = scope_with_size.from(from)
+        response = scope.search(:search_type => 'query_then_fetch')
+        populate_counts(response)
+        yield materialize_hits(response['hits']['hits'])
+        from += size
+        @count ||= scope.count
+      end while from < @count
+    end
+
+    def scan_in_batches(batch_options, &block)
+      batch_options = batch_options.symbolize_keys
+      scroll_options = {
+        :scroll => "#{batch_options[:ttl] || 60}s",
+        :size => batch_options[:batch_size] || ::Elastictastic.config.default_batch_size
+      }
+      scan_response = ::Elastictastic.client.search(
+        @index,
+        @clazz.type,
+        params,
+        scroll_options.merge(:search_type => 'scan')
+      )
+
+      @count = scan_response['hits']['total']
+      scroll_id = scan_response['_scroll_id']
+
+      begin
+        response = ::Elastictastic.client.scroll(scroll_id, scroll_options.slice(:scroll))
+        scroll_id = response['_scroll_id']
+        yield materialize_hits(response['hits']['hits'])
+      end until response['hits']['hits'].empty?
+    end
+
+    def populate_counts(response = nil)
+      response ||= search(:search_type => 'count')
+      @count ||= response['hits']['total']
+      if response['facets']
+        @all_facets ||= ::Hashie::Mash.new(response['facets'])
+      end
+    end
+
+    def find_one(id)
+      data = ::Elastictastic.client.get(index, type, id, params_for_find_one)
+      return nil if data['exists'] == false
+      case data['status']
+      when nil
+        materialize_hit(data)
+      when 404
+        nil
+      end
+    end
+
+    def find_many(ids)
+      docspec = ids.map do |id|
+        { '_id' => id }.merge!(params_for_find_many)
+      end
+      materialize_hits(
+        ::Elastictastic.client.mget(docspec, index, type)['docs']
+      ).map { |result, hit| result }
+    end
+
+    def find_many_in_many_indices(ids_by_index)
+      docs = []
+      ids_by_index.each_pair do |index, ids|
+        ::Kernel.Array(ids).each do |id|
+          docs << doc = {
+            '_id' => id.to_s,
+            '_type' => type,
+            '_index' => index
+          }
+          doc['fields'] = ::Kernel.Array(@search['fields']) if @search['fields']
+        end
+      end
+      materialize_hits(
+        ::Elastictastic.client.mget(docs)['docs']
+      ).map { |result, hit| result }
+    end
+
+    def params_for_find_one
+      params_for_find.tap do |params|
+        params['fields'] &&= params['fields'].join(',')
+      end
+    end
+
+    def params_for_find_many
+      params_for_find
+    end
+
+    def params_for_find
+      {}.tap do |params|
+        params['fields'] = ::Kernel.Array(@search['fields']) if @search['fields']
+      end
+    end
+
+    def materialize_hits(hits)
+      unless ::Kernel.block_given?
+        return ::Enumerator.new(self, :materialize_hits, hits)
+      end
+      hits.each do |hit|
+        unless hit['exists'] == false
+          yield materialize_hit(hit), ::Hashie::Mash.new(hit)
+        end
+      end
+    end
+
+    def materialize_hit(hit)
+      @clazz.new.tap do |result|
+        result.parent_collection = @parent_collection if @parent_collection
+        result.elasticsearch_hit = hit
+      end
+    end
+  end
+end

data/lib/elastictastic/scope_builder.rb

@@ -0,0 +1,32 @@
+module Elastictastic
+  class ScopeBuilder < BasicObject
+    class <<self
+      private :new
+
+      def build(&block)
+        new(&block).build
+      end
+    end
+
+    def initialize(&block)
+      @block = block
+    end
+
+    def build
+      @scope = {}
+      instance_eval(&@block)
+      @scope
+    end
+
+    def method_missing(method, *args, &block)
+      args << ScopeBuilder.build(&block) if block
+      value =
+        case args.length
+        when 0 then {}
+        when 1 then args.first
+        else args
+        end
+      @scope[method.to_s] = value
+    end
+  end
+end

data/lib/elastictastic/scoped.rb

@@ -0,0 +1,20 @@
+module Elastictastic
+  module Scoped
+    def with_scope(scope)
+      scope_stack.push(scope)
+      begin
+        yield
+      ensure
+        scope_stack.pop
+      end
+    end
+
+    def scope_stack
+      Thread.current["#{name}::scope_stack"] ||= []
+    end
+
+    def current_scope
+      scope_stack.last || default_scope
+    end
+  end
+end