es-index 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f83238da2c0540e6b9f128ceb77fb1c83c1572fa
+  data.tar.gz: 8c5b6f6b5fc5ca1de2a86233d066f1c99d2d3324
+SHA512:
+  metadata.gz: be74797a59e73804f81225753070e22f9a752c357ebb1ed3a1e1069b86d63d85eae506a84580cdb5752b925687422bf644025624b6ff29f2e748ea3076c697e9
+  data.tar.gz: 9c7a291ccd232628f7056eef1791fab4396b48b250e837ba78e86940d35358fee3dbe513c365d3f1551f248c28f5f0764699ea615ac0d27242405d190a4c40c2

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.swp
+*.swo

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in es-index.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Cary Dunn
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+Intended to be used with toy-dynamo and dynamodb
+
+# ES::Index
+
+es_index_model do
+  es_index "services"
+  es_type "service"
+  es_ttl do |service|
+    15.minutes
+  end
+  es_id do |service|
+    service.guid
+  end
+
+  es_mapping do
+    # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html
+    {
+      name: { type: 'string', analyzer: 'standard' }
+      guid: { type: 'string', include_in_all: false }
+    }
+  end
+
+  to_es_json do |service|
+    {
+      name: service.name,
+      guid: service.guid,
+      description: service.description,
+      account_guid: service.account_guid,
+      user_guid: service.user_guid,
+      is_public: service.is_public
+    }
+  end
+end
+
+Model.es_search
+  returns ES::Index::SearchResponse

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/es-index.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'es/index/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "es-index"
+  spec.version       = ES::Index::VERSION
+  spec.authors       = ["Cary Dunn"]
+  spec.email         = ["cary.dunn@gmail.com"]
+  spec.description   = "Elasticsearch + toy-dynamo"
+  spec.summary       = "Elasticsearch + toy-dynamo"
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+
+  spec.add_dependency "hashie", ">= 2.0"
+end

data/lib/es/index/client.rb

@@ -0,0 +1,9 @@
+module ES
+  module Index
+    module Client
+      def self.connection
+        @connection ||= Elasticsearch::Client.new(ES::Index::Config.elasticsearch_config)
+      end
+    end
+  end
+end

data/lib/es/index/config/options.rb

@@ -0,0 +1,44 @@
+module ES
+  module Index
+    module Config
+
+      # Encapsulates logic for setting options.
+      module Options
+
+        def defaults
+          @defaults ||= {}
+        end
+
+        def option(name, options = {})
+          defaults[name] = settings[name] = options[:default]
+
+          class_eval <<-RUBY
+            def #{name}
+              settings[#{name.inspect}]
+            end
+
+            def #{name}=(value)
+              settings[#{name.inspect}] = value
+            end
+
+            def #{name}?
+              #{name}
+            end
+            
+            def reset_#{name}
+              settings[#{name.inspect}] = defaults[#{name.inspect}]
+            end
+          RUBY
+        end
+
+        def reset
+          settings.replace(defaults)
+        end
+
+        def settings
+          @settings ||= {}
+        end
+      end
+    end
+  end
+end

data/lib/es/index/config.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require "uri"
+require "es/index/config/options"
+
+module ES
+  module Index
+    module Config
+      extend self
+      extend Options
+
+      # All the default options.
+      option :logger, :default => defined?(Rails)
+      # https://github.com/elasticsearch/elasticsearch-ruby/blob/master/elasticsearch-transport/lib/elasticsearch/transport/client.rb
+      option :elasticsearch_config
+
+      def default_logger
+        defined?(Rails) && Rails.respond_to?(:logger) ? Rails.logger : ::Logger.new($stdout)
+      end
+
+      def logger
+        @logger ||= default_logger
+      end
+
+      def logger=(logger)
+        case logger
+        when false, nil then @logger = nil
+        when true then @logger = default_logger
+        else
+          @logger = logger if logger.respond_to?(:info)
+        end
+      end
+
+    end
+  end
+end
+

data/lib/es/index/model.rb

@@ -0,0 +1,244 @@
+require "es/index/client"
+
+module ES
+  module Index
+    module Model
+      extend ActiveSupport::Concern
+
+      included do
+        after_save lambda { 
+          update_es_index
+        }
+
+        after_destroy lambda { 
+          update_es_index
+        }
+
+        def update_es_index(options={})
+          if self.class.es_if.nil? || self.class.es_if.call
+            index_req = {
+              index: (options[:index_name] || self.class.es_index),
+              type: self.class.es_type,
+              id: self.class.es_id.call(self),
+              body: self.class.to_es_json.call(self),
+            }
+            index_req.merge!(:ttl => self.class.es_ttl.call(self)) if self.class.es_ttl
+            ES::Index::Client.connection.index(index_req)
+          end
+        end
+      end
+
+      # Add neccessary infrastructure for the model, when missing in
+      # some half-baked ActiveModel implementations.
+      #
+      #if base.respond_to?(:before_destroy) && !base.instance_methods.map(&:to_sym).include?(:destroyed?)
+        #base.class_eval do
+          #before_destroy  { @destroyed = true }
+          #def destroyed?; !!@destroyed; end
+        #end
+      #end
+
+      module ClassMethods
+
+        def es
+          ES::Index::Client.connection
+        end
+        
+        def es_index_exists?(options={})
+          ES::Index::Client.connection.indices.exists({ index: (options[:index_name] || es_index) })
+        end
+
+        #https://github.com/elasticsearch/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/indices/create.rb
+        #client.indices.delete index: 'foo*'
+        def es_delete_index(options={})
+          ES::Index::Client.connection.indices.delete({ index: (options[:index_name] || es_index) })
+        end
+
+        def es_create_index(options={})
+          es_delete_index if es_index_exists?
+          index_req_body = {
+            settings: es_settings,
+            analysis: es_analysis,
+            mappings: {
+              es_type.to_sym => {
+                properties: es_mapping
+              }
+            }
+          }
+          ES::Index::Client.connection.indices.create(
+            index: (options[:index_name] || es_index),
+            body: index_req_body
+          )
+        end
+
+        def es_index_model(options={}, &block)
+          if block
+            @es_index_block ||= block
+          else
+            unless @es_index_configged
+              @es_index_block.call
+              @es_index_configged = true
+            end
+          end
+        end
+
+        def es_mapping(&block)
+          @es_mapping ||= {}
+          if block
+            @es_mapping = block.call
+          else
+            self.es_index_model
+            @es_mapping
+          end
+        end
+
+        def es_analysis(&block)
+          @es_analysis ||= {}
+          if block
+            @es_analysis = block.call
+          else
+            self.es_index_model
+            @es_analysis
+          end
+        end
+
+        def es_settings(&block)
+          @es_settings ||= {}
+          if block
+            @es_settings = block.call
+          else
+            self.es_index_model
+            @es_settings
+          end
+        end
+
+        def es_type(val=nil)
+          if val
+            @es_type = val
+          else
+            self.es_index_model
+            @es_type || self.name.to_s.underscore
+          end
+        end
+
+        def es_index(val=nil)
+          if val
+            @es_index = val
+          else
+            self.es_index_model
+            @es_index || self.name.to_s.underscore.pluralize
+          end
+        end
+
+        def es_ttl(&block)
+          if block
+            @es_ttl = block
+          else
+            self.es_index_model
+            @es_ttl
+          end
+        end
+
+        def es_if(&block)
+          if block
+            @es_if = block
+          else
+            self.es_index_model
+            @es_if
+          end
+        end
+
+        def es_id(&block)
+          @es_id ||= lambda {|o| o.id }
+          if block
+            @es_id = block
+          else
+            self.es_index_model
+            @es_id
+          end
+        end
+
+        def es_results_to_models(&block)
+          if block
+            @es_results_to_models = block
+          else
+            self.es_index_model
+            @es_results_to_models
+          end
+        end
+
+        def to_es_json(&block)
+          @to_es_json ||= lambda {|o| { id: o.id } }
+          if block
+            @to_es_json = block
+          else
+            self.es_index_model
+            @to_es_json
+          end
+        end
+
+        # :batch_size
+        # :index_name
+        # :scan_filter
+        def es_import(options={})
+          unless self.es_index_exists?(options)
+            self.es_create_index(options)
+          end
+
+          # Use 1/4 or read provision
+          read_provision = self.dynamo_table.table_schema[:provisioned_throughput][:read_capacity_units]
+          raise "read_provision not set for class!" unless read_provision
+          default_batch_size = (self.read_provision / 2.0).floor
+          batch_size = options[:batch_size] || default_batch_size
+          puts "Indexing via scan with batch size of #{batch_size}..."
+
+          # :consumed_capacity
+          scan_idx = 0
+          results_hash = {}
+          while scan_idx == 0 || (results_hash && results_hash[:last_evaluated_key])
+            puts "Batch iteration #{scan_idx+1}..."
+            scan_options = {:batch => batch_size, :manual_batching => true, :return_consumed_capacity => :total}
+            scan_options.merge!(:exclusive_start_key => results_hash[:last_evaluated_key]) if results_hash[:last_evaluated_key]
+            scan_options.merge!(:scan_filter => options[:scan_filter]) if options[:scan_filter]
+            results_hash = self.scan(scan_options)
+            unless results_hash[:results].blank?
+              puts "Indexing #{results_hash[:results].size} results..."
+              results_hash[:results].each do |r|
+                r.update_es_index(options)
+              end
+            end
+
+            # If more results to scan, sleep to throttle...
+            #   Local Dynamo is not returning consumed_capacity 2014-01-12
+            if results_hash[:last_evaluated_key] && results_hash[:consumed_capacity]
+              # try to keep read usage under 50% of read_provision
+              sleep_time = results_hash[:consumed_capacity][:capacity_units].to_f / (read_provision / 2.0)
+              puts "Sleeping for #{sleep_time}..."
+              sleep(sleep_time)
+            end
+            
+            scan_idx += 1
+          end
+        end
+
+        def es_search(body={}, options={})
+          SearchResponse.new(ES::Index::Client.connection.search({
+            index: (options[:index_name] || self.es_index),
+            type: (options[:type] || self.es_type),
+            body: body
+          }), self)
+        end
+
+        def es_suggest(body={}, options={})
+          SearchResponse.new(ES::Index::Client.connection.suggest({
+            index: (options[:index_name] || self.es_index),
+            type: (options[:type] || self.es_type),
+            body: body
+          }), self)
+        end
+
+      end # ClassMethods
+
+    end
+  end
+end

data/lib/es/index/search_response.rb

@@ -0,0 +1,91 @@
+require 'hashie'
+
+module ES
+  module Index
+    #{
+    #       "took" => 0,
+    #  "timed_out" => false,
+    #    "_shards" => {
+    #             "total" => 5,
+    #        "successful" => 5,
+    #            "failed" => 0
+    #    },
+    #       "hits" => {
+    #              "total" => 2468,
+    #          "max_score" => 1.0,
+    #               "hits" => [
+    #              [0] {
+    #                   "_index" => "services",
+    #                    "_type" => "service",
+    #                      "_id" => "b86ac976-5c5b-11e3-9470-cc5bbc629773",
+    #                   "_score" => 1.0,
+    #                  "_source" => {
+    #                              "name" => "...",
+    #                              "guid" => "b86ac976-5c5b-11e3-9470-cc5bbc629773",
+    #                       "description" => nil,
+    #                      "account_guid" => "77c57a7f-6ff3-4918-8f61-ef5ae5a04e18",
+    #                         "user_guid" => "3cc66723-f788-40d7-8583-bfa09cec5623",
+    #                            "domain" => nil,
+    #                         "is_public" => false
+    #                  }
+    #              }
+    #          ]
+    #      }
+    #  }
+    class SearchResponse
+
+      attr_accessor :raw_response
+      attr_accessor :model
+
+      def initialize(response, model)
+        @raw_response = HashResponse.new(response)
+        @model = model
+        self
+      end
+
+      def hits
+        HashResponse.new(@raw_response.hits)
+      end
+
+      def total
+        hits.total
+      end
+
+      def max_score
+        hits.max_score
+      end
+
+      def results
+        hits.hits
+      end
+
+      def models
+        @model.es_results_to_models.call(results)
+      end
+
+      def took
+        @raw_response.took
+      end
+
+      def timed_out
+        @raw_response.timed_out
+      end
+
+      def shards
+        @raw_response._shards
+      end
+
+      def method_missing(method, *args, &block)
+        @raw_response.send(method, *args, &block)
+      end
+
+    end
+
+    class HashResponse < Hash
+      include Hashie::Extensions::MethodAccess
+      include Hashie::Extensions::MergeInitializer
+      include Hashie::Extensions::IndifferentAccess
+    end
+  end
+end
+

data/lib/es/index/tasks.rb

@@ -0,0 +1,89 @@
+require 'rake'
+
+module ES
+  module Index
+    module Tasks
+      extend self
+      def included_models
+        dir = ENV['DIR'].to_s != '' ? ENV['DIR'] : Rails.root.join("app/models")
+        puts "Loading models from: #{dir}"
+        included = []
+        Dir.glob(File.join("#{dir}/**/*.rb")).each do |path|
+          model_filename = path[/#{Regexp.escape(dir.to_s)}\/([^\.]+).rb/, 1]
+          next if model_filename.match(/^concerns\//i) # Skip concerns/ folder
+          klass = model_filename.camelize.constantize
+
+          begin
+            klass = model_filename.camelize.constantize
+          rescue NameError
+            require(path) ? retry : raise
+          end
+
+          # Skip if the class doesn't have Toy::Dynamo integration
+          next unless klass.respond_to?(:dynamo_table)
+
+          included << klass
+        end
+        included
+      end
+    end
+  end
+end
+
+namespace :es do
+  desc 'Create ElasticSearch Index'
+  task :create => :environment do
+    raise "expected usage: rake es:create CLASS=Service" unless ENV['CLASS']
+    options = {}
+    options.merge!(:index_name => ENV['INDEX']) if ENV['INDEX']
+    if ENV["CLASS"] == "all"
+      ES::Index::Tasks.included_models.each do |klass|
+        puts "Creating index for #{klass}..."
+        begin
+          klass.es_create_index(options)
+        rescue Exception => e
+          puts "Could not index table! #{e.inspect}"
+        end
+      end
+    else
+      ENV['CLASS'].constantize.es_create_index(options)
+    end
+  end
+
+  desc 'Destroy ElasticSearch Index'
+  task :destroy => :environment do
+    raise "expected usage: rake es:destroy CLASS=Service" unless ENV['CLASS']
+    options = {}
+    options.merge!(:index_name => ENV['INDEX']) if ENV['INDEX']
+    if ENV["CLASS"] == "all"
+      ES::Index::Tasks.included_models.each do |klass|
+        puts "Destroying index for #{klass}..."
+        begin
+          klass.es_delete_index(options)
+        rescue Exception => e
+          puts "Could not index table! #{e.inspect}"
+        end
+      end
+    else
+      ENV['CLASS'].constantize.es_delete_index(options)
+    end
+  end
+
+  # TODO: Parallel scan support
+  # TODO: Add ability to define custom methods for retrieving data to index (ie. using hash + range instead of scan to limit amount of scanning)
+  desc "Reindex ElasticSearch"
+  task :reindex => :environment do
+    raise "expected usage: rake es:reindex CLASS=Service" unless ENV['CLASS']
+    options = {}
+    options.merge!(:index_name => ENV['INDEX']) if ENV['INDEX']
+
+    klass = ENV['CLASS'].constantize
+    batch_size = ENV['BATCH']
+
+    klass.es_import({
+      :index_name => ENV["INDEX"],
+      :batch_size => batch_size
+    })
+  end
+
+end

data/lib/es/index/version.rb

@@ -0,0 +1,5 @@
+module ES
+  module Index
+    VERSION = "0.0.2"
+  end
+end

data/lib/es/index.rb

@@ -0,0 +1,20 @@
+require "es/index/version"
+require "es/index/model"
+require "es/index/search_response"
+require "es/index/config"
+require "es/index/tasks"
+
+module ES
+  module Index
+    extend self
+
+    def configure
+      block_given? ? yield(ES::Index::Config) : ES::Index::Config
+    end
+    alias :config :configure
+
+    def logger
+      ES::Index::Config.logger
+    end
+  end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: es-index
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Cary Dunn
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: Elasticsearch + toy-dynamo
+email:
+- cary.dunn@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- es-index.gemspec
+- lib/es/index.rb
+- lib/es/index/client.rb
+- lib/es/index/config.rb
+- lib/es/index/config/options.rb
+- lib/es/index/model.rb
+- lib/es/index/search_response.rb
+- lib/es/index/tasks.rb
+- lib/es/index/version.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: Elasticsearch + toy-dynamo
+test_files: []