elastics 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 31208a61d960f7b411e4b6588d8c7aa2f8741e59
+  data.tar.gz: 64d46d44db05a22ac1d84a5155cb5ea6489327aa
+SHA512:
+  metadata.gz: 998636cd31b1a9229d381544206fb3a17e4c7bef3ba6d4b26935e7d7254e55ab545a8247668063a61b64979115d3332cb1bd7f83fa1c95d3123ec1ab10b3b987
+  data.tar.gz: 7be4cb9870cd5cc3e053f492fd55479e1ab41287790047987b9897430d8dddab3ba2b8e3689038cd04e8361a00383d082fd2b66b1625cce82a734b584abf0ec1

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,19 @@
+PATH
+  remote: .
+  specs:
+    elastics (0.1.0)
+      httpclient (~> 2.4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    httpclient (2.4.0)
+    rake (10.3.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.5)
+  elastics!
+  rake (~> 10)

data/README.md

@@ -0,0 +1,103 @@
+# elastics
+
+Simple ElasticSearch client.
+
+Fast and thread-safe [httpclient](https://github.com/nahi/httpclient) under the hood.
+
+## Install
+
+```ruby
+# Gemfile
+gem 'elastics', github: 'printercu/elastics-rb'
+```
+
+## Usage
+
+### Plain
+
+```ruby
+# initialize client with
+client = Elastics::Client.new(options)
+# options is hash with
+#   :host
+#   :port
+#   :index  - (default index)
+#   :type   - (default type)
+
+# basic request
+client.request(options)
+# options is hash with
+#   :method - default :get
+#   :data   - post body
+#   :query  - query string params
+#   :index, :type, :id - query path params to override defaults
+
+# method shortcuts for #put, #post #delete
+client.delete(params)
+
+# getter/setter shortcuts
+client.set(id, data)
+client.get(id)
+client.get(params) # as usual
+
+# other shortcuts (set method & id)
+client.put_mapping(index: index, type: type, data: mapping)
+client.search(params)
+client.index(params) # PUT if :id is set, otherwise POST
+
+# utils
+client.index_exists?(name)
+```
+
+### ActiveRecord
+
+```ruby
+class User < ActiveRecord::Base
+  indexed_with_elastics
+  # optionally pass :index, :type
+
+  # optionally override to export only selected fields
+  def to_elastics
+    serializable_hash(only: [:id, :first_name, :last_name])
+  end
+end
+
+User.search_elastics(data)
+```
+
+#### Configure
+```yml
+# database.yml
+development:
+  elastics:
+    # use single index (app_dev/users, app_dev/documents)
+    index: app_dev
+
+    # use index per type (app_dev_users/users, app_dev_documents/documents)
+    index_prefix: app_dev_
+
+production:
+  elastics:
+    host: 10.0.0.1
+    port: 1234
+
+    index: app
+    # or
+    index_prefix: app_
+```
+
+#### Mappings & index settings
+Mappings & index settings `.yml` files are placed in
+`db/elastics/mappings` & `db/elastics/indices`.
+For now this files are not related to models and only used by rake tasks.
+
+- `rake elastics:create` (or `Elastics::Tasks.create_indices`)
+creates index with settings for each file from `indices` folder.
+For single index it only processes file with index name.
+For multiple indices each index name is `#{index_prefix}#{file.basename}`
+
+- `rake elastics:migrate` (or `Elastics::Tasks.migrate`)
+puts mappings from `mappings` folder.
+
+## License
+MIT

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'lib/elastics'
+  t.libs << 'test'
+  t.test_files = FileList['test/lib/elastics/*_test.rb']
+  t.verbose = true
+end
+
+task default: :test

data/elastics.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'elastics/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'elastics'
+  spec.version       = Elastics::VERSION
+  spec.authors       = ['Max Melentiev']
+  spec.email         = ['melentievm@gmail.com']
+  spec.summary       = 'ElasticSearch client with ActiveRecord integration'
+  spec.description   = 'Lightweight and extensible elasticsearch client'
+  spec.homepage      = 'http://github.com/printercu/elastics-rb'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'httpclient', '~> 2.4.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake', '~> 10'
+end

data/lib/elastics/active_record/helper_methods.rb

@@ -0,0 +1,59 @@
+module Elastics
+  module ActiveRecord
+    module HelperMethods
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def search(data = {}, routing = nil)
+          es_results = search_elastics(data, routing)
+          ids = es_results['hits'.freeze]['hits'.freeze].map { |x| x['_id'.freeze].to_i }
+          relation = where(id: ids)
+          items_by_id = relation.index_by(&:id)
+          collection = ids.map { |i| items_by_id[i] }
+          {
+            collection: collection,
+            relation: relation,
+            search: es_results,
+          }
+        end
+
+        def search_elastics(data = {}, routing = nil)
+          request = {
+            id: :_search,
+            data: data,
+          }
+          request[:query] = {routing: routing} if routing
+          request_elastics(request)
+        end
+
+        def request_elastics(params)
+          request = {
+            index:  elastics_index_name,
+            type:   elastics_type_name,
+          }.merge!(params)
+          elastics.request(request)
+        end
+
+        def elastics_mapping
+          request_elastics(method: :get, id: :_mapping)
+        end
+
+        def reindex(*args)
+          find_each(*args, &:index_elastics)
+        end
+      end
+
+      def index_elastics
+        self.class.request_elastics(method: :post, id: id, data: to_elastics)
+      end
+
+      def delete_elastics
+        self.class.request_elastics(method: :delete, id: id)
+      end
+
+      def to_elastics
+        as_json
+      end
+    end
+  end
+end

data/lib/elastics/active_record/instrumentation.rb

@@ -0,0 +1,15 @@
+module Elastics
+  module ActiveRecord
+    module Instrumentation
+      def request_elastics(params = {})
+        data = {
+          name:     name,
+          request:  params,
+        }
+        ActiveSupport::Notifications.instrument 'request_elastics.active_record', data do
+          super(params)
+        end
+      end
+    end
+  end
+end

data/lib/elastics/active_record/log_subscriber.rb

@@ -0,0 +1,27 @@
+module Elastics
+  module ActiveRecord
+    module LogSubscriber
+      def self.included(base)
+        instance_methods.each { |method| base.method_added(method) }
+      end
+
+      def request_elastics(event)
+        return unless logger.debug?
+
+        payload = event.payload
+
+        name    = "#{payload[:name]} elastics (#{event.duration.round(1)}ms)"
+        request = payload[:request].to_json
+
+        if odd?
+          name    = color(name, ActiveSupport::LogSubscriber::CYAN, true)
+          request = color(request, nil, true)
+        else
+          name = color(name, ActiveSupport::LogSubscriber::MAGENTA, true)
+        end
+
+        debug "  #{name}  #{request}"
+      end
+    end
+  end
+end

data/lib/elastics/active_record/model_schema.rb

@@ -0,0 +1,39 @@
+module Elastics
+  module ActiveRecord
+    module ModelSchema
+      attr_writer :elastics_index_name, :elastics_type_name
+
+      def elastics_index_name
+        reset_elastics_index_name unless defined?(@elastics_index_name)
+        @elastics_index_name
+      end
+
+      def elastics_type_name
+        reset_elastics_index_name unless defined?(@elastics_type_name)
+        @elastics_type_name
+      end
+
+      def reset_elastics_index_name
+        superclass_responds = superclass.respond_to?(:elastics_index_name)
+        index = if abstract_class? && superclass_responds
+          superclass == ::ActiveRecord::Base ? nil : superclass.elastics_index_name
+        elsif superclass.abstract_class? && superclass_responds
+          superclass.elastics_index_name || compute_elastics_index_name
+        else
+          compute_elastics_index_name
+        end
+        @elastics_index_name = index
+        @elastics_type_name = compute_elastics_type_name
+      end
+
+      def compute_elastics_index_name(name = nil)
+        elastics_config[:index] ||
+          "#{elastics_config[:index_prefix]}#{name || table_name.singularize}"
+      end
+
+      def compute_elastics_type_name
+        model_name.to_s.demodulize.underscore.singularize
+      end
+    end
+  end
+end

data/lib/elastics/active_record.rb

@@ -0,0 +1,36 @@
+module Elastics
+  module ActiveRecord
+    extend ActiveSupport::Autoload
+
+    autoload :ModelSchema
+    autoload :HelperMethods
+    autoload :Instrumentation
+    autoload :LogSubscriber
+
+    def elastics_config
+      @elastics_config ||= connection_config[:elastics].try!(:with_indifferent_access) ||
+        raise('No elastics configuration in database.yml')
+    end
+
+    def elastics
+      @elastics ||= Client.new elastics_config.slice(:host, :port)
+    end
+
+    def indexed_with_elastics(options = {})
+      options = {
+        hooks: [:update, :destroy],
+      }.merge(options)
+
+      extend ModelSchema
+      include HelperMethods
+      extend Instrumentation
+
+      self.elastics_index_name  = options[:index] if options[:index]
+      self.elastics_type_name   = options[:type]  if options[:type]
+
+      hooks = options[:hooks]
+      after_commit :index_elastics, on: [:create, :update] if hooks.include?(:update)
+      after_commit :delete_elastics, on: [:destroy] if hooks.include?(:destroy)
+    end
+  end
+end

data/lib/elastics/client.rb

@@ -0,0 +1,97 @@
+require 'httpclient'
+
+module Elastics
+  class Client
+    HEADERS = {'Content-Type' => 'application/json'}
+
+    attr_writer :index, :type
+    attr_reader :client
+
+    def initialize(defaults = {})
+      @host = defaults[:host] || '127.0.0.1'
+      @port = defaults[:port] || 9200
+      @index  = defaults[:index]
+      @type   = defaults[:type]
+      @client = HTTPClient.new
+    end
+
+    def debug!(dev = STDOUT)
+      @client.debug_dev = dev
+    end
+
+    def debug_off!
+      @client.debug_dev = nil
+    end
+
+    def set_index(index, type = nil)
+      @index = index || nil
+      @type  = type  || nil
+    end
+
+    def uri(params)
+      str = "http://#{@host}:#{@port}"
+      if index = params[:index]  || @index
+        str += "/#{index}"
+        type  = params[:type]   || @type
+        str += "/#{type}" if type
+      end
+      path = params[:id]
+      str += "/#{path}" if path
+      str
+    end
+
+    def request(params)
+      http_method = params[:method] || :get
+      body = params[:data].try!(:to_json)
+      res = @client.request(http_method, uri(params), params[:query], body, HEADERS)
+      status = res.status
+      return JSON.parse(res.body) if 300 > status
+      if 404 == status
+        message = JSON.parse(res.body)['error'] rescue 'Not found'
+        raise NotFound, message
+      end
+      raise Error.new(res.reason)
+    end
+
+    # shortcuts
+    [:put, :post, :delete].each do |method|
+      define_method(method) do |params|
+        params[:method] = method
+        request params
+      end
+    end
+
+    def get(params)
+      params = {id: params} unless params.is_a?(Hash)
+      params[:method] = :get
+      request(params)
+    end
+
+    def set(id, data)
+      request(id: id, data: data, method: :put)
+    end
+
+    def put_mapping(params)
+      params[:id] = :_mapping
+      params[:method] = :put
+      request(params)
+    end
+
+    def search(params)
+      params[:id] = :_search
+      params[:method] = :post
+      request(params)
+    end
+
+    def index(params)
+      params[:id] ? put(params) : post(params)
+    end
+
+    def index_exists?(index)
+      get(index: index, type: nil, id: :_mapping)
+      true
+    rescue NotFound
+      false
+    end
+  end
+end

data/lib/elastics/query_helper.rb

@@ -0,0 +1,27 @@
+module Elastics
+  module QueryHelper
+    def normalize_filters(filters)
+      return filters unless filters.is_a?(Array)
+      return filters[0] if 2 > filters.size
+      {and: {filters: filters}}
+    end
+
+    def normalize_query(query, filters)
+      filter = normalize_filters filters
+      query ||= {match_all: {}}
+      return query unless filter
+      {filtered: {
+        query:  query,
+        filter: filter,
+      }}
+    end
+
+    def terms_query(field, val, options = {})
+      if val.is_a?(Array)
+        {terms: {field => val}.merge(options)}
+      else
+        result = {term: {field => val}}
+      end
+    end
+  end
+end

data/lib/elastics/railtie.rb

@@ -0,0 +1,18 @@
+require 'elastics/active_record'
+
+module Elastics
+  class Railtie < Rails::Railtie
+    initializer 'elastics.configure_rails_initialization' do
+      ::ActiveRecord::Base.extend Elastics::ActiveRecord
+      unless ::ActiveRecord::LogSubscriber < ActiveRecord::LogSubscriber
+        ::ActiveRecord::LogSubscriber.send :include, ActiveRecord::LogSubscriber
+      end
+    end
+
+    rake_tasks do
+      load 'tasks/elastics.rake'
+    end
+  end
+end
+
+Elastics::Railtie.run_initializers

data/lib/elastics/tasks/indices.rb

@@ -0,0 +1,42 @@
+module Elastics
+  module Tasks
+    module Indices
+      attr_writer :indices_path
+
+      def indices_paths
+        @indices_paths ||= base_paths.map { |x| File.join x, 'indices' }
+      end
+
+      def indices_settings
+        @indices_settings ||= indices_paths.map { |path| Dir["#{path}/*.yml"] }.
+          flatten.sort.
+          each_with_object({}) do |file, hash|
+            name = File.basename file, '.yml'
+            data = YAML.load_file(file)
+            name = "#{config[:index_prefix]}#{name}"
+            hash[name] = data[Rails.env] || data
+          end
+      end
+
+      def indices
+        @indices ||= config[:index] ? [config[:index]] : indices_settings.keys
+      end
+
+      def delete_indices
+        indices.each do |index|
+          log "Delete index #{index}"
+          client.delete index: index rescue NotFound
+        end
+      end
+
+      def create_indices
+        indices.each do |index|
+          log "Create index #{index}"
+          unless client.index_exists?(index)
+            client.put(index: index, data: indices_settings[index])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elastics/tasks/mappings.rb

@@ -0,0 +1,40 @@
+module Elastics
+  module Tasks
+    module Mappings
+      attr_writer :mappings_path
+
+      def mappings_paths
+        @mappings_paths ||= base_paths.map { |x| File.join x, 'mappings' }
+      end
+
+      def put_mappings
+        mappings.each do |type, mapping|
+          index = index_for_type(type)
+          log "Put mapping #{index}/#{type}"
+          client.put_mapping index: index, type: type, data: mapping
+        end
+      end
+
+      def mappings
+        @mappings ||= mappings_paths.map { |path| Dir["#{path}/*.yml"] }.
+          flatten.sort.
+          each_with_object({}) do |file, hash|
+            name = File.basename file, '.yml'
+            hash[name] = YAML.load_file(file)
+          end
+      end
+
+      def types
+        @types ||= mappings.keys
+      end
+
+      def indices
+        @indices ||= (super + types.map { |type| index_for_type(type) }).uniq
+      end
+
+      def index_for_type(type)
+        config[:index] || "#{config[:index_prefix]}#{type}"
+      end
+    end
+  end
+end

data/lib/elastics/tasks.rb

@@ -0,0 +1,35 @@
+module Elastics
+  module Tasks
+    require 'elastics/tasks/indices'
+    include Indices
+
+    require 'elastics/tasks/mappings'
+    include Mappings
+
+    extend self
+
+    attr_writer :base_paths, :client, :config
+
+    def base_paths
+      @base_paths ||= [File.join(Rails.root, 'db', 'elastics')]
+    end
+
+    def migrate(options = {})
+      delete_indices if options[:flush]
+      create_indices
+      put_mappings
+    end
+
+    def client
+      @client ||= ::ActiveRecord::Base.elastics
+    end
+
+    def config
+      @config ||= ::ActiveRecord::Base.elastics_config
+    end
+
+    def log(*args)
+      Rails.logger.info(*args)
+    end
+  end
+end

data/lib/elastics/version.rb

@@ -0,0 +1,7 @@
+module Elastics
+  VERSION = '0.1.0'
+
+  def self.gem_version
+    Gem::Version.new VERSION
+  end
+end

data/lib/elastics.rb

@@ -0,0 +1,13 @@
+module Elastics
+  class Error < StandardError; end
+  class NotFound < Error; end
+
+  require 'elastics/client'
+  require 'elastics/query_helper'
+
+  autoload :Tasks, 'elastics/tasks'
+
+  extend QueryHelper
+end
+
+require 'elastics/railtie' if defined?(Rails)

data/lib/tasks/elastics.rake

@@ -0,0 +1,20 @@
+namespace 'elastics' do
+  task load_config: [:environment, 'db:load_config'] do
+  end
+
+  desc 'Creates indices and applies mappings (Use NOFLUSH to prevent old indices from deletion)'
+  task migrate: :load_config do
+    flush = !ENV.key?('NOFLUSH')
+    Elastics::Tasks.migrate(flush: flush)
+  end
+
+  desc 'Creates indices'
+  task create: :load_config do
+    Elastics::Tasks.create_indices
+  end
+
+  desc 'Drops indices'
+  task delete: :load_config do
+    Elastics::Tasks.delete_indices
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: elastics
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Max Melentiev
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.4.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+description: Lightweight and extensible elasticsearch client
+email:
+- melentievm@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- elastics.gemspec
+- lib/elastics.rb
+- lib/elastics/active_record.rb
+- lib/elastics/active_record/helper_methods.rb
+- lib/elastics/active_record/instrumentation.rb
+- lib/elastics/active_record/log_subscriber.rb
+- lib/elastics/active_record/model_schema.rb
+- lib/elastics/client.rb
+- lib/elastics/query_helper.rb
+- lib/elastics/railtie.rb
+- lib/elastics/tasks.rb
+- lib/elastics/tasks/indices.rb
+- lib/elastics/tasks/mappings.rb
+- lib/elastics/version.rb
+- lib/tasks/elastics.rake
+homepage: http://github.com/printercu/elastics-rb
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: ElasticSearch client with ActiveRecord integration
+test_files: []