elastic-rails 0.1.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4b7fcb53e08a79f313f4d3a4209f600ca621efd4
-  data.tar.gz: 082420e99a70e5d5ddd8b037e0f27eeec77b1363
+  metadata.gz: a5c4d94d6d4b35572694f0968dfcb60251cd531a
+  data.tar.gz: 8fc9e4861325fb3e7ddf90d5751b7dd69892a0ac
 SHA512:
-  metadata.gz: e2f2356d768d8ebf6b97aca4d6cc42929bc2c57716df72a03ddc416444aec1ec46a4a7bc60522d08a7ae581f906f4a402c1a5129d66f61472f2d8732df8db67f
-  data.tar.gz: 25a21fe7f5d976aae28e26b0e98b173ed38add01102b38793ff1c31b7b2786b5ecd7fde2ab57e7d6d471e42a47bf0ec9c0ea1caf0ced1c48d7a5c67753e6a7cc
+  metadata.gz: 7084256b0dca89eecfdb1a0b2b2c8784e52b155da96fc60b9419913314fe7fb06c01525669a10414cb807e0ec75d6538d8d0c7f344d14eb6fdde16abe3f8442c
+  data.tar.gz: 7f38c177624c2df6216e821701507a91000bd95980226bdad50bf1fab18d6146f1041bd4ffc1845c6824bfd819a8d6b31cba12b789abfd05d875ef13efe7418b

data/Guardfile

@@ -0,0 +1,46 @@
+  watch(%r{^spec/.+_spec\.rb$})
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec feature)
+
+## Uncomment to clear the screen before every task
+# clearing :on
+
+## Make Guard exit when config is changed so it can be restarted
+#
+## Note: if you want Guard to automatically start up again, run guard in a
+## shell loop, e.g.:
+#
+#  $ while bundle exec guard; do echo "Restarting Guard..."; done
+#
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), the you will want to move the Guardfile
+## to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+#
+watch ("Guardfile") do
+  UI.info "Exiting because Guard must be restarted for changes to take effect"
+  exit 0
+end
+
+# Note: The cmd option is now required due to the increasing number of ways
+#       rspec may be run, below are examples of the most common uses.
+#  * bundler: 'bundle exec rspec'
+#  * bundler binstubs: 'bin/rspec'
+#  * spring: 'bin/rspec' (This will use spring if running and you have
+#                          installed the spring binstubs per the docs)
+#  * zeus: 'zeus rspec' (requires the server to be started separately)
+#  * 'just' rspec: 'rspec'
+
+guard :rspec, cmd: 'bundle exec rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/elastic/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/elastic.gemspec

@@ -19,9 +19,18 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "elasticsearch", "~> 1.0"
+  spec.add_dependency "elasticsearch", "~> 1.0", ">= 1.0.18"
+  spec.add_dependency "activesupport"
 
-  spec.add_development_dependency "bundler", "~> 1.12"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake", "~> 10.4"
+  spec.add_development_dependency "rspec", "~> 3.1"
+  spec.add_development_dependency "rspec-nc", "~> 0.2"
+  spec.add_development_dependency "guard", "~> 2.11"
+  spec.add_development_dependency "guard-rspec", "~> 4.5"
+  spec.add_development_dependency "terminal-notifier-guard", "~> 1.6", ">= 1.6.1"
+  spec.add_development_dependency "pry", "~> 0.10"
+  spec.add_development_dependency "pry-remote", "~> 0.1"
+  spec.add_development_dependency "pry-byebug", "~> 3.2"
+  spec.add_development_dependency "pry-nav", "~> 0.2"
 end

data/lib/elastic/commands/build_agg_from_params.rb

@@ -0,0 +1,63 @@
+module Elastic::Commands
+  class BuildAggFromParams < Elastic::Support::Command.new(:index, :params)
+    def perform
+      parse_params
+      raise ArgumentError, "field not mapped: #{@field}" unless index.definition.has_field? @field
+
+      path = parse_nesting_path
+      raise NotImplementedError, "nested paths not yet supported in aggregations" if path
+
+      build_node
+    end
+
+    private
+
+    def agg_name
+      @options.fetch(:as, @field)
+    end
+
+    def parse_params
+      @field = params[0].to_s
+      @options = params[1] || {}
+    end
+
+    def parse_nesting_path
+      dot_index = @field.rindex('.')
+      return nil if dot_index.nil?
+      @field.slice(0, dot_index)
+    end
+
+    def build_node
+      type = @options[:type]
+      type = infer_type_from_options_and_mapping if type.nil?
+
+      send("build_#{type}")
+    end
+
+    def infer_type_from_options_and_mapping
+      return :range if @options.key? :ranges
+
+      properties = index.mapping.get_field_options @field
+      return :date_histogram if properties['type'] == 'date'
+      return :histogram if @options.key? :interval
+
+      :terms
+    end
+
+    def build_range
+      raise NotImplementedError, 'range aggregation not yet implemented'
+    end
+
+    def build_histogram
+      raise NotImplementedError, 'histogram aggregation not yet implemented'
+    end
+
+    def build_date_histogram
+      Elastic::Nodes::Agg::DateHistogram.build(agg_name, @field, interval: @options[:interval])
+    end
+
+    def build_terms
+      Elastic::Nodes::Agg::Terms.build(agg_name, @field, size: @options[:size])
+    end
+  end
+end

data/lib/elastic/commands/build_query_from_params.rb

@@ -0,0 +1,132 @@
+module Elastic::Commands
+  class BuildQueryFromParams < Elastic::Support::Command.new(
+    :index, :params, block: nil, prefix: nil
+  )
+
+    def perform
+      if block
+        # TODO: builder mode, support nesting through first parameter
+      else
+        node = Elastic::Nodes::Boolean.build_or(params.map do |part|
+          Elastic::Nodes::Boolean.build_and(part.map do |field, options|
+            field = field.to_s
+            path = get_nesting_path field
+            query_node = build_query_node(field, options)
+            path.nil? ? query_node : Elastic::Nodes::Nested.build(with_prefix(path), query_node)
+          end)
+        end)
+      end
+
+      node.simplify
+    end
+
+    private
+
+    def get_nesting_path(_field)
+      dot_index = _field.rindex('.')
+      return nil if dot_index.nil?
+      _field.slice 0, dot_index
+    end
+
+    def build_query_node(_field, _options)
+      _field = with_prefix _field
+      _options = prepare_options _field, _options
+
+      type = infer_type_from_params(_options)
+      send("build_#{type}", _field, _options)
+    end
+
+    def infer_type_from_params(_query)
+      return :term if _query.key? :term
+      return :term if _query.key? :terms
+      return :match if _query.key? :matches
+      return :range if _query.key? :gte
+      return :range if _query.key? :gt
+      return :range if _query.key? :lte
+      return :range if _query.key? :lt
+      return :nested if _query.key? :nested
+      nil
+    end
+
+    def with_prefix(_path)
+      return _path if prefix.nil?
+      "#{prefix}.#{_path}"
+    end
+
+    def prepare_options(_field, _options)
+      properties = index.mapping.get_field_options _field.to_s
+      raise ArgumentError, "field not mapped: #{_field}" if properties.nil?
+
+      case properties['type']
+      when 'nested'
+        prepare_nested_options _options, properties
+      when 'string'
+        prepare_string_options _options, properties
+      else
+        prepare_default_options _options, properties
+      end
+    end
+
+    def prepare_nested_options(_options, _properties)
+      return _options if _options.is_a?(Hash) && _options[:nested]
+      { nested: _options }
+    end
+
+    def prepare_string_options(_options, _properties)
+      return _options if _options.is_a? Hash
+      _properties['index'] == 'not_analyzed' ? { term: _options } : { matches: _options }
+    end
+
+    def prepare_default_options(_options, _properties)
+      return _options if _options.is_a? Hash
+      return range_to_options(_options) if _options.is_a? Range
+      { term: _options }
+    end
+
+    def build_nested(_field, _options)
+      query = _options[:nested]
+      query = [query] unless query.is_a? Array
+
+      nested_query = BuildQueryFromParams.for(index: index, params: query, prefix: _field)
+      Elastic::Nodes::Nested.build _field, nested_query
+    end
+
+    def build_term(_field, _options)
+      terms = Array(_options[:term] || _options[:terms])
+
+      Elastic::Nodes::Term.new.tap do |node|
+        node.field = _field
+        node.mode = _options[:mode]
+        node.terms = terms.map { |t| prep(_field, t) }
+      end
+    end
+
+    def build_range(_field, _options)
+      Elastic::Nodes::Range.new.tap do |node|
+        node.field = _field
+        node.gte = prep(_field, _options[:gte]) if _options.key? :gte
+        node.gt = prep(_field, _options[:gt]) if _options.key? :gt
+        node.lte = prep(_field, _options[:lte]) if _options.key? :lte
+        node.lt = prep(_field, _options[:lt]) if _options.key? :lt
+      end
+    end
+
+    def build_match(_field, _options)
+      Elastic::Nodes::Match.new.tap do |node|
+        node.field = _field
+        node.query = prep(_field, _options[:matches])
+      end
+    end
+
+    def range_to_options(_range)
+      {
+        gte: _range.begin,
+        (_range.exclude_end? ? :lt : :lte) => _range.end
+      }
+    end
+
+    def prep(_field, _value)
+      index.definition.get_field(_field).prepare_value_for_query(_value)
+    end
+  end
+end

data/lib/elastic/commands/import_index_documents.rb

@@ -0,0 +1,63 @@
+module Elastic::Commands
+  class ImportIndexDocuments < Elastic::Support::Command.new(
+    :index, collection: nil, cache_size: 10000, verbose: false
+  )
+    def perform
+      if collection.present?
+        import_collection
+      else
+        targets.each { |target| import_target(target) }
+      end
+      flush
+    end
+
+    private
+
+    def import_collection
+      main_target.collect_for(collection, middleware_options) { |obj| queue obj }
+    end
+
+    def import_target(_target)
+      _target.collect_all(middleware_options) { |obj| queue obj }
+    end
+
+    def cache
+      @cache ||= []
+    end
+
+    def queue(_object)
+      cache << render_for_es(_object)
+      flush if cache.length >= cache_size
+    end
+
+    def flush
+      unless cache.empty?
+        index.adaptor.bulk_index(cache)
+        log_flush(cache.size) if verbose
+        cache.clear
+      end
+    end
+
+    def log_flush(_size)
+      @total ||= 0
+      @total += _size
+      Elastic::Configuration.logger.info "Imported #{@total} documents"
+    end
+
+    def render_for_es(_object)
+      index.new(_object).as_es_document
+    end
+
+    def main_target
+      index.definition.main_target
+    end
+
+    def targets
+      index.definition.targets
+    end
+
+    def middleware_options
+      index.definition.middleware_options
+    end
+  end
+end

data/lib/elastic/configuration.rb

@@ -0,0 +1,61 @@
+module Elastic
+  module Configuration
+    extend self
+
+    def configure(_options = nil, &_block)
+      if _options.nil?
+        _block.call self
+      else
+        @config = config.merge _options.symbolize_keys
+      end
+    end
+
+    def api_client
+      config[:client] ||= load_api_client
+    end
+
+    def index_name
+      config[:index]
+    end
+
+    def indices_path
+      'app/indices'
+    end
+
+    def page_size
+      @config[:page_size]
+    end
+
+    def coord_similarity
+      @config[:coord_similarity]
+    end
+
+    def strict_mode
+      @config[:strict_types]
+    end
+
+    def logger
+      @config[:logger] || default_logger
+    end
+
+    private
+
+    def config
+      @config ||= {
+        host: '127.0.0.1',
+        port: 9200,
+        page_size: 20,
+        coord_similarity: true,
+        strict_types: true
+      }
+    end
+
+    def default_logger
+      @default_logger ||= Logger.new(STDOUT)
+    end
+
+    def load_api_client
+      Elasticsearch::Client.new host: config[:host], port: config[:port]
+    end
+  end
+end

data/lib/elastic/core/adaptor.rb

@@ -0,0 +1,102 @@
+module Elastic::Core
+  class Adaptor
+    def initialize(_suffix)
+      @suffix = _suffix
+    end
+
+    def index_name
+      @index_name ||= "#{Elastic::Configuration.index_name}_#{@suffix}"
+    end
+
+    def remap(_type, _mapping)
+      # TODO
+    end
+
+    def exists?
+      api_client.indices.exists? build_options
+    end
+
+    def ensure_index
+      create unless exists?
+      self
+    end
+
+    def create
+      api_client.indices.create build_options
+      api_client.cluster.health wait_for_status: 'yellow'
+      self
+    end
+
+    def drop
+      api_client.indices.delete build_options
+      self
+    end
+
+    def exists_type?(_type)
+      api_client.indices.exists_type build_options(type: _type)
+    end
+
+    def exists_mapping?(_type)
+      !api_client.indices.get_mapping(build_options(type: _type)).empty?
+    end
+
+    def get_mappings(type: nil)
+      mappings = api_client.indices.get_mapping build_options(type: type)
+      mappings[index_name]['mappings']
+    end
+
+    def set_mapping(_type, _mapping)
+      api_client.indices.put_mapping build_options(
+        type: _type,
+        update_all_types: true,
+        body: _mapping
+      )
+      self
+    end
+
+    def index(_document)
+      api_client.index build_options(
+        id: _document['_id'],
+        type: _document['_type'],
+        body: _document['data']
+      )
+      self
+    end
+
+    def bulk_index(_documents)
+      body = _documents.map do |doc|
+        { 'index' => doc.merge('_index' => index_name) }
+      end
+
+      api_client.bulk body: body
+      self
+    end
+
+    def refresh
+      api_client.indices.refresh build_options
+      self
+    end
+
+    def find(_id, type: '_all')
+      api_client.get build_options(type: type, id: _id)
+    end
+
+    def count(type: nil, query: nil)
+      api_client.count(build_options(type: type, body: query))['count']
+    end
+
+    def query(type: nil, query: nil)
+      api_client.search build_options(type: type, body: query)
+    end
+
+    private
+
+    def api_client
+      Elastic::Configuration.api_client
+    end
+
+    def build_options(_options = {})
+      { index: index_name }.merge! _options
+    end
+  end
+end

data/lib/elastic/core/base_middleware.rb

@@ -0,0 +1,43 @@
+module Elastic::Core
+  class BaseMiddleware
+    attr_reader :target
+
+    def initialize(_target)
+      @target = _target
+    end
+
+    def type_name
+      @target.to_s
+    end
+
+    def mode
+      not_supported :mode
+    end
+
+    def field_options_for(_field, _options)
+      not_supported :field_options_for
+    end
+
+    def collect_all(_options, &_block)
+      not_supported :collect_all
+    end
+
+    def collect_for(_collection, _options, &_block)
+      not_supported :collect_for
+    end
+
+    def find_by_ids(_ids, _options)
+      not_supported :find_by_ids
+    end
+
+    def build_from_data(_data, _options)
+      not_supported :build_from_data
+    end
+
+    private
+
+    def not_supported(_feature)
+      raise NotImplementedError, "#{self.class} does not support '#{_feature}'"
+    end
+  end
+end

data/lib/elastic/core/default_middleware.rb

@@ -0,0 +1,64 @@
+module Elastic::Core
+  class DefaultMiddleware < BaseMiddleware
+    def mode
+      case target_mode
+      when :find_multiple_ids, :find_single_id
+        :index
+      else
+        :storage
+      end
+    end
+
+    def field_options_for(_field, _options)
+      nil
+    end
+
+    def collect_all(_options, &_block)
+      method = collect_method_for(target)
+      target.public_send(method, &_block) if method
+    end
+
+    def collect_for(_collection, _options, &_block)
+      method = collect_method_for(_collection)
+      raise ArgumentError, "Could not find a method to iterate over collection" if method.nil?
+      _collection.public_send(method, &_block)
+    end
+
+    def find_by_ids(_ids, _options)
+      case target_mode
+      when :find_multiple_ids
+        target.find_by_elastic_ids(_ids)
+      when :find_single_id
+        _ids.map { |id| target.find_by_elastic_id(id) }
+      end
+    end
+
+    def build_from_data(_data, _options)
+      case target_mode
+      when :custom_build
+        target.build_from_elastic_data(_data)
+      when :open_struct
+        OpenStruct.new _data
+      end
+    end
+
+    private
+
+    def target_mode
+      @target_mode ||= detect_output_mode
+    end
+
+    def collect_method_for(_target)
+      return :find_each_for_elastic if _target.respond_to?(:find_each_for_elastic)
+      return :each if _target.respond_to?(:each)
+      nil
+    end
+
+    def detect_output_mode
+      return :find_multiple_ids if target.respond_to? :find_by_elastic_ids
+      return :find_single_id if target.respond_to? :find_by_elastic_id
+      return :custom_build if target.respond_to? :build_from_elastic_data
+      :open_struct
+    end
+  end
+end