es-index 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +36 -0
- data/Rakefile +1 -0
- data/es-index.gemspec +25 -0
- data/lib/es/index/client.rb +9 -0
- data/lib/es/index/config/options.rb +44 -0
- data/lib/es/index/config.rb +36 -0
- data/lib/es/index/model.rb +244 -0
- data/lib/es/index/search_response.rb +91 -0
- data/lib/es/index/tasks.rb +89 -0
- data/lib/es/index/version.rb +5 -0
- data/lib/es/index.rb +20 -0
- metadata +100 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f83238da2c0540e6b9f128ceb77fb1c83c1572fa
|
4
|
+
data.tar.gz: 8c5b6f6b5fc5ca1de2a86233d066f1c99d2d3324
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: be74797a59e73804f81225753070e22f9a752c357ebb1ed3a1e1069b86d63d85eae506a84580cdb5752b925687422bf644025624b6ff29f2e748ea3076c697e9
|
7
|
+
data.tar.gz: 9c7a291ccd232628f7056eef1791fab4396b48b250e837ba78e86940d35358fee3dbe513c365d3f1551f248c28f5f0764699ea615ac0d27242405d190a4c40c2
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 Cary Dunn
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
Intended to be used with toy-dynamo and dynamodb
|
2
|
+
|
3
|
+
# ES::Index
|
4
|
+
|
5
|
+
es_index_model do
|
6
|
+
es_index "services"
|
7
|
+
es_type "service"
|
8
|
+
es_ttl do |service|
|
9
|
+
15.minutes
|
10
|
+
end
|
11
|
+
es_id do |service|
|
12
|
+
service.guid
|
13
|
+
end
|
14
|
+
|
15
|
+
es_mapping do
|
16
|
+
# http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html
|
17
|
+
{
|
18
|
+
name: { type: 'string', analyzer: 'standard' }
|
19
|
+
guid: { type: 'string', include_in_all: false }
|
20
|
+
}
|
21
|
+
end
|
22
|
+
|
23
|
+
to_es_json do |service|
|
24
|
+
{
|
25
|
+
name: service.name,
|
26
|
+
guid: service.guid,
|
27
|
+
description: service.description,
|
28
|
+
account_guid: service.account_guid,
|
29
|
+
user_guid: service.user_guid,
|
30
|
+
is_public: service.is_public
|
31
|
+
}
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
Model.es_search
|
36
|
+
returns ES::Index::SearchResponse
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/es-index.gemspec
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'es/index/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "es-index"
|
8
|
+
spec.version = ES::Index::VERSION
|
9
|
+
spec.authors = ["Cary Dunn"]
|
10
|
+
spec.email = ["cary.dunn@gmail.com"]
|
11
|
+
spec.description = "Elasticsearch + toy-dynamo"
|
12
|
+
spec.summary = "Elasticsearch + toy-dynamo"
|
13
|
+
spec.homepage = ""
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
22
|
+
spec.add_development_dependency "rake"
|
23
|
+
|
24
|
+
spec.add_dependency "hashie", ">= 2.0"
|
25
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module ES
|
2
|
+
module Index
|
3
|
+
module Config
|
4
|
+
|
5
|
+
# Encapsulates logic for setting options.
|
6
|
+
module Options
|
7
|
+
|
8
|
+
def defaults
|
9
|
+
@defaults ||= {}
|
10
|
+
end
|
11
|
+
|
12
|
+
def option(name, options = {})
|
13
|
+
defaults[name] = settings[name] = options[:default]
|
14
|
+
|
15
|
+
class_eval <<-RUBY
|
16
|
+
def #{name}
|
17
|
+
settings[#{name.inspect}]
|
18
|
+
end
|
19
|
+
|
20
|
+
def #{name}=(value)
|
21
|
+
settings[#{name.inspect}] = value
|
22
|
+
end
|
23
|
+
|
24
|
+
def #{name}?
|
25
|
+
#{name}
|
26
|
+
end
|
27
|
+
|
28
|
+
def reset_#{name}
|
29
|
+
settings[#{name.inspect}] = defaults[#{name.inspect}]
|
30
|
+
end
|
31
|
+
RUBY
|
32
|
+
end
|
33
|
+
|
34
|
+
def reset
|
35
|
+
settings.replace(defaults)
|
36
|
+
end
|
37
|
+
|
38
|
+
def settings
|
39
|
+
@settings ||= {}
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "uri"
|
3
|
+
require "es/index/config/options"
|
4
|
+
|
5
|
+
module ES
|
6
|
+
module Index
|
7
|
+
module Config
|
8
|
+
extend self
|
9
|
+
extend Options
|
10
|
+
|
11
|
+
# All the default options.
|
12
|
+
option :logger, :default => defined?(Rails)
|
13
|
+
# https://github.com/elasticsearch/elasticsearch-ruby/blob/master/elasticsearch-transport/lib/elasticsearch/transport/client.rb
|
14
|
+
option :elasticsearch_config
|
15
|
+
|
16
|
+
def default_logger
|
17
|
+
defined?(Rails) && Rails.respond_to?(:logger) ? Rails.logger : ::Logger.new($stdout)
|
18
|
+
end
|
19
|
+
|
20
|
+
def logger
|
21
|
+
@logger ||= default_logger
|
22
|
+
end
|
23
|
+
|
24
|
+
def logger=(logger)
|
25
|
+
case logger
|
26
|
+
when false, nil then @logger = nil
|
27
|
+
when true then @logger = default_logger
|
28
|
+
else
|
29
|
+
@logger = logger if logger.respond_to?(:info)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
@@ -0,0 +1,244 @@
|
|
1
|
+
require "es/index/client"
|
2
|
+
|
3
|
+
module ES
|
4
|
+
module Index
|
5
|
+
module Model
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
included do
|
9
|
+
after_save lambda {
|
10
|
+
update_es_index
|
11
|
+
}
|
12
|
+
|
13
|
+
after_destroy lambda {
|
14
|
+
update_es_index
|
15
|
+
}
|
16
|
+
|
17
|
+
def update_es_index(options={})
|
18
|
+
if self.class.es_if.nil? || self.class.es_if.call
|
19
|
+
index_req = {
|
20
|
+
index: (options[:index_name] || self.class.es_index),
|
21
|
+
type: self.class.es_type,
|
22
|
+
id: self.class.es_id.call(self),
|
23
|
+
body: self.class.to_es_json.call(self),
|
24
|
+
}
|
25
|
+
index_req.merge!(:ttl => self.class.es_ttl.call(self)) if self.class.es_ttl
|
26
|
+
ES::Index::Client.connection.index(index_req)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
# Add neccessary infrastructure for the model, when missing in
|
32
|
+
# some half-baked ActiveModel implementations.
|
33
|
+
#
|
34
|
+
#if base.respond_to?(:before_destroy) && !base.instance_methods.map(&:to_sym).include?(:destroyed?)
|
35
|
+
#base.class_eval do
|
36
|
+
#before_destroy { @destroyed = true }
|
37
|
+
#def destroyed?; !!@destroyed; end
|
38
|
+
#end
|
39
|
+
#end
|
40
|
+
|
41
|
+
module ClassMethods
|
42
|
+
|
43
|
+
def es
|
44
|
+
ES::Index::Client.connection
|
45
|
+
end
|
46
|
+
|
47
|
+
def es_index_exists?(options={})
|
48
|
+
ES::Index::Client.connection.indices.exists({ index: (options[:index_name] || es_index) })
|
49
|
+
end
|
50
|
+
|
51
|
+
#https://github.com/elasticsearch/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/indices/create.rb
|
52
|
+
#client.indices.delete index: 'foo*'
|
53
|
+
def es_delete_index(options={})
|
54
|
+
ES::Index::Client.connection.indices.delete({ index: (options[:index_name] || es_index) })
|
55
|
+
end
|
56
|
+
|
57
|
+
def es_create_index(options={})
|
58
|
+
es_delete_index if es_index_exists?
|
59
|
+
index_req_body = {
|
60
|
+
settings: es_settings,
|
61
|
+
analysis: es_analysis,
|
62
|
+
mappings: {
|
63
|
+
es_type.to_sym => {
|
64
|
+
properties: es_mapping
|
65
|
+
}
|
66
|
+
}
|
67
|
+
}
|
68
|
+
ES::Index::Client.connection.indices.create(
|
69
|
+
index: (options[:index_name] || es_index),
|
70
|
+
body: index_req_body
|
71
|
+
)
|
72
|
+
end
|
73
|
+
|
74
|
+
def es_index_model(options={}, &block)
|
75
|
+
if block
|
76
|
+
@es_index_block ||= block
|
77
|
+
else
|
78
|
+
unless @es_index_configged
|
79
|
+
@es_index_block.call
|
80
|
+
@es_index_configged = true
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def es_mapping(&block)
|
86
|
+
@es_mapping ||= {}
|
87
|
+
if block
|
88
|
+
@es_mapping = block.call
|
89
|
+
else
|
90
|
+
self.es_index_model
|
91
|
+
@es_mapping
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def es_analysis(&block)
|
96
|
+
@es_analysis ||= {}
|
97
|
+
if block
|
98
|
+
@es_analysis = block.call
|
99
|
+
else
|
100
|
+
self.es_index_model
|
101
|
+
@es_analysis
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def es_settings(&block)
|
106
|
+
@es_settings ||= {}
|
107
|
+
if block
|
108
|
+
@es_settings = block.call
|
109
|
+
else
|
110
|
+
self.es_index_model
|
111
|
+
@es_settings
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def es_type(val=nil)
|
116
|
+
if val
|
117
|
+
@es_type = val
|
118
|
+
else
|
119
|
+
self.es_index_model
|
120
|
+
@es_type || self.name.to_s.underscore
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def es_index(val=nil)
|
125
|
+
if val
|
126
|
+
@es_index = val
|
127
|
+
else
|
128
|
+
self.es_index_model
|
129
|
+
@es_index || self.name.to_s.underscore.pluralize
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
def es_ttl(&block)
|
134
|
+
if block
|
135
|
+
@es_ttl = block
|
136
|
+
else
|
137
|
+
self.es_index_model
|
138
|
+
@es_ttl
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def es_if(&block)
|
143
|
+
if block
|
144
|
+
@es_if = block
|
145
|
+
else
|
146
|
+
self.es_index_model
|
147
|
+
@es_if
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
def es_id(&block)
|
152
|
+
@es_id ||= lambda {|o| o.id }
|
153
|
+
if block
|
154
|
+
@es_id = block
|
155
|
+
else
|
156
|
+
self.es_index_model
|
157
|
+
@es_id
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def es_results_to_models(&block)
|
162
|
+
if block
|
163
|
+
@es_results_to_models = block
|
164
|
+
else
|
165
|
+
self.es_index_model
|
166
|
+
@es_results_to_models
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
def to_es_json(&block)
|
171
|
+
@to_es_json ||= lambda {|o| { id: o.id } }
|
172
|
+
if block
|
173
|
+
@to_es_json = block
|
174
|
+
else
|
175
|
+
self.es_index_model
|
176
|
+
@to_es_json
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
# :batch_size
|
181
|
+
# :index_name
|
182
|
+
# :scan_filter
|
183
|
+
def es_import(options={})
|
184
|
+
unless self.es_index_exists?(options)
|
185
|
+
self.es_create_index(options)
|
186
|
+
end
|
187
|
+
|
188
|
+
# Use 1/4 or read provision
|
189
|
+
read_provision = self.dynamo_table.table_schema[:provisioned_throughput][:read_capacity_units]
|
190
|
+
raise "read_provision not set for class!" unless read_provision
|
191
|
+
default_batch_size = (self.read_provision / 2.0).floor
|
192
|
+
batch_size = options[:batch_size] || default_batch_size
|
193
|
+
puts "Indexing via scan with batch size of #{batch_size}..."
|
194
|
+
|
195
|
+
# :consumed_capacity
|
196
|
+
scan_idx = 0
|
197
|
+
results_hash = {}
|
198
|
+
while scan_idx == 0 || (results_hash && results_hash[:last_evaluated_key])
|
199
|
+
puts "Batch iteration #{scan_idx+1}..."
|
200
|
+
scan_options = {:batch => batch_size, :manual_batching => true, :return_consumed_capacity => :total}
|
201
|
+
scan_options.merge!(:exclusive_start_key => results_hash[:last_evaluated_key]) if results_hash[:last_evaluated_key]
|
202
|
+
scan_options.merge!(:scan_filter => options[:scan_filter]) if options[:scan_filter]
|
203
|
+
results_hash = self.scan(scan_options)
|
204
|
+
unless results_hash[:results].blank?
|
205
|
+
puts "Indexing #{results_hash[:results].size} results..."
|
206
|
+
results_hash[:results].each do |r|
|
207
|
+
r.update_es_index(options)
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
# If more results to scan, sleep to throttle...
|
212
|
+
# Local Dynamo is not returning consumed_capacity 2014-01-12
|
213
|
+
if results_hash[:last_evaluated_key] && results_hash[:consumed_capacity]
|
214
|
+
# try to keep read usage under 50% of read_provision
|
215
|
+
sleep_time = results_hash[:consumed_capacity][:capacity_units].to_f / (read_provision / 2.0)
|
216
|
+
puts "Sleeping for #{sleep_time}..."
|
217
|
+
sleep(sleep_time)
|
218
|
+
end
|
219
|
+
|
220
|
+
scan_idx += 1
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
def es_search(body={}, options={})
|
225
|
+
SearchResponse.new(ES::Index::Client.connection.search({
|
226
|
+
index: (options[:index_name] || self.es_index),
|
227
|
+
type: (options[:type] || self.es_type),
|
228
|
+
body: body
|
229
|
+
}), self)
|
230
|
+
end
|
231
|
+
|
232
|
+
def es_suggest(body={}, options={})
|
233
|
+
SearchResponse.new(ES::Index::Client.connection.suggest({
|
234
|
+
index: (options[:index_name] || self.es_index),
|
235
|
+
type: (options[:type] || self.es_type),
|
236
|
+
body: body
|
237
|
+
}), self)
|
238
|
+
end
|
239
|
+
|
240
|
+
end # ClassMethods
|
241
|
+
|
242
|
+
end
|
243
|
+
end
|
244
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'hashie'
|
2
|
+
|
3
|
+
module ES
|
4
|
+
module Index
|
5
|
+
#{
|
6
|
+
# "took" => 0,
|
7
|
+
# "timed_out" => false,
|
8
|
+
# "_shards" => {
|
9
|
+
# "total" => 5,
|
10
|
+
# "successful" => 5,
|
11
|
+
# "failed" => 0
|
12
|
+
# },
|
13
|
+
# "hits" => {
|
14
|
+
# "total" => 2468,
|
15
|
+
# "max_score" => 1.0,
|
16
|
+
# "hits" => [
|
17
|
+
# [0] {
|
18
|
+
# "_index" => "services",
|
19
|
+
# "_type" => "service",
|
20
|
+
# "_id" => "b86ac976-5c5b-11e3-9470-cc5bbc629773",
|
21
|
+
# "_score" => 1.0,
|
22
|
+
# "_source" => {
|
23
|
+
# "name" => "...",
|
24
|
+
# "guid" => "b86ac976-5c5b-11e3-9470-cc5bbc629773",
|
25
|
+
# "description" => nil,
|
26
|
+
# "account_guid" => "77c57a7f-6ff3-4918-8f61-ef5ae5a04e18",
|
27
|
+
# "user_guid" => "3cc66723-f788-40d7-8583-bfa09cec5623",
|
28
|
+
# "domain" => nil,
|
29
|
+
# "is_public" => false
|
30
|
+
# }
|
31
|
+
# }
|
32
|
+
# ]
|
33
|
+
# }
|
34
|
+
# }
|
35
|
+
class SearchResponse
|
36
|
+
|
37
|
+
attr_accessor :raw_response
|
38
|
+
attr_accessor :model
|
39
|
+
|
40
|
+
def initialize(response, model)
|
41
|
+
@raw_response = HashResponse.new(response)
|
42
|
+
@model = model
|
43
|
+
self
|
44
|
+
end
|
45
|
+
|
46
|
+
def hits
|
47
|
+
HashResponse.new(@raw_response.hits)
|
48
|
+
end
|
49
|
+
|
50
|
+
def total
|
51
|
+
hits.total
|
52
|
+
end
|
53
|
+
|
54
|
+
def max_score
|
55
|
+
hits.max_score
|
56
|
+
end
|
57
|
+
|
58
|
+
def results
|
59
|
+
hits.hits
|
60
|
+
end
|
61
|
+
|
62
|
+
def models
|
63
|
+
@model.es_results_to_models.call(results)
|
64
|
+
end
|
65
|
+
|
66
|
+
def took
|
67
|
+
@raw_response.took
|
68
|
+
end
|
69
|
+
|
70
|
+
def timed_out
|
71
|
+
@raw_response.timed_out
|
72
|
+
end
|
73
|
+
|
74
|
+
def shards
|
75
|
+
@raw_response._shards
|
76
|
+
end
|
77
|
+
|
78
|
+
def method_missing(method, *args, &block)
|
79
|
+
@raw_response.send(method, *args, &block)
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
83
|
+
|
84
|
+
class HashResponse < Hash
|
85
|
+
include Hashie::Extensions::MethodAccess
|
86
|
+
include Hashie::Extensions::MergeInitializer
|
87
|
+
include Hashie::Extensions::IndifferentAccess
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'rake'
|
2
|
+
|
3
|
+
module ES
|
4
|
+
module Index
|
5
|
+
module Tasks
|
6
|
+
extend self
|
7
|
+
def included_models
|
8
|
+
dir = ENV['DIR'].to_s != '' ? ENV['DIR'] : Rails.root.join("app/models")
|
9
|
+
puts "Loading models from: #{dir}"
|
10
|
+
included = []
|
11
|
+
Dir.glob(File.join("#{dir}/**/*.rb")).each do |path|
|
12
|
+
model_filename = path[/#{Regexp.escape(dir.to_s)}\/([^\.]+).rb/, 1]
|
13
|
+
next if model_filename.match(/^concerns\//i) # Skip concerns/ folder
|
14
|
+
klass = model_filename.camelize.constantize
|
15
|
+
|
16
|
+
begin
|
17
|
+
klass = model_filename.camelize.constantize
|
18
|
+
rescue NameError
|
19
|
+
require(path) ? retry : raise
|
20
|
+
end
|
21
|
+
|
22
|
+
# Skip if the class doesn't have Toy::Dynamo integration
|
23
|
+
next unless klass.respond_to?(:dynamo_table)
|
24
|
+
|
25
|
+
included << klass
|
26
|
+
end
|
27
|
+
included
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
namespace :es do
|
34
|
+
desc 'Create ElasticSearch Index'
|
35
|
+
task :create => :environment do
|
36
|
+
raise "expected usage: rake es:create CLASS=Service" unless ENV['CLASS']
|
37
|
+
options = {}
|
38
|
+
options.merge!(:index_name => ENV['INDEX']) if ENV['INDEX']
|
39
|
+
if ENV["CLASS"] == "all"
|
40
|
+
ES::Index::Tasks.included_models.each do |klass|
|
41
|
+
puts "Creating index for #{klass}..."
|
42
|
+
begin
|
43
|
+
klass.es_create_index(options)
|
44
|
+
rescue Exception => e
|
45
|
+
puts "Could not index table! #{e.inspect}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
else
|
49
|
+
ENV['CLASS'].constantize.es_create_index(options)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
desc 'Destroy ElasticSearch Index'
|
54
|
+
task :destroy => :environment do
|
55
|
+
raise "expected usage: rake es:destroy CLASS=Service" unless ENV['CLASS']
|
56
|
+
options = {}
|
57
|
+
options.merge!(:index_name => ENV['INDEX']) if ENV['INDEX']
|
58
|
+
if ENV["CLASS"] == "all"
|
59
|
+
ES::Index::Tasks.included_models.each do |klass|
|
60
|
+
puts "Destroying index for #{klass}..."
|
61
|
+
begin
|
62
|
+
klass.es_delete_index(options)
|
63
|
+
rescue Exception => e
|
64
|
+
puts "Could not index table! #{e.inspect}"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
else
|
68
|
+
ENV['CLASS'].constantize.es_delete_index(options)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# TODO: Parallel scan support
|
73
|
+
# TODO: Add ability to define custom methods for retrieving data to index (ie. using hash + range instead of scan to limit amount of scanning)
|
74
|
+
desc "Reindex ElasticSearch"
|
75
|
+
task :reindex => :environment do
|
76
|
+
raise "expected usage: rake es:reindex CLASS=Service" unless ENV['CLASS']
|
77
|
+
options = {}
|
78
|
+
options.merge!(:index_name => ENV['INDEX']) if ENV['INDEX']
|
79
|
+
|
80
|
+
klass = ENV['CLASS'].constantize
|
81
|
+
batch_size = ENV['BATCH']
|
82
|
+
|
83
|
+
klass.es_import({
|
84
|
+
:index_name => ENV["INDEX"],
|
85
|
+
:batch_size => batch_size
|
86
|
+
})
|
87
|
+
end
|
88
|
+
|
89
|
+
end
|
data/lib/es/index.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require "es/index/version"
|
2
|
+
require "es/index/model"
|
3
|
+
require "es/index/search_response"
|
4
|
+
require "es/index/config"
|
5
|
+
require "es/index/tasks"
|
6
|
+
|
7
|
+
module ES
|
8
|
+
module Index
|
9
|
+
extend self
|
10
|
+
|
11
|
+
def configure
|
12
|
+
block_given? ? yield(ES::Index::Config) : ES::Index::Config
|
13
|
+
end
|
14
|
+
alias :config :configure
|
15
|
+
|
16
|
+
def logger
|
17
|
+
ES::Index::Config.logger
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
metadata
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: es-index
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.2
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Cary Dunn
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-03-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.3'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.3'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: hashie
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '2.0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '2.0'
|
55
|
+
description: Elasticsearch + toy-dynamo
|
56
|
+
email:
|
57
|
+
- cary.dunn@gmail.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- .gitignore
|
63
|
+
- Gemfile
|
64
|
+
- LICENSE.txt
|
65
|
+
- README.md
|
66
|
+
- Rakefile
|
67
|
+
- es-index.gemspec
|
68
|
+
- lib/es/index.rb
|
69
|
+
- lib/es/index/client.rb
|
70
|
+
- lib/es/index/config.rb
|
71
|
+
- lib/es/index/config/options.rb
|
72
|
+
- lib/es/index/model.rb
|
73
|
+
- lib/es/index/search_response.rb
|
74
|
+
- lib/es/index/tasks.rb
|
75
|
+
- lib/es/index/version.rb
|
76
|
+
homepage: ''
|
77
|
+
licenses:
|
78
|
+
- MIT
|
79
|
+
metadata: {}
|
80
|
+
post_install_message:
|
81
|
+
rdoc_options: []
|
82
|
+
require_paths:
|
83
|
+
- lib
|
84
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - '>='
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
90
|
+
requirements:
|
91
|
+
- - '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
requirements: []
|
95
|
+
rubyforge_project:
|
96
|
+
rubygems_version: 2.1.11
|
97
|
+
signing_key:
|
98
|
+
specification_version: 4
|
99
|
+
summary: Elasticsearch + toy-dynamo
|
100
|
+
test_files: []
|