elastics-client 1.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +20 -0
- data/README.md +28 -0
- data/VERSION +1 -0
- data/elastics-client.gemspec +33 -0
- data/lib/elastics.rb +108 -0
- data/lib/elastics/api_stubs.rb +1268 -0
- data/lib/elastics/api_templates/cluster_api.yml +94 -0
- data/lib/elastics/api_templates/core_api.yml +202 -0
- data/lib/elastics/api_templates/indices_api.yml +304 -0
- data/lib/elastics/class_proxy/base.rb +29 -0
- data/lib/elastics/class_proxy/templates.rb +97 -0
- data/lib/elastics/class_proxy/templates/doc.rb +91 -0
- data/lib/elastics/class_proxy/templates/search.rb +72 -0
- data/lib/elastics/configuration.rb +25 -0
- data/lib/elastics/deprecation.rb +153 -0
- data/lib/elastics/errors.rb +43 -0
- data/lib/elastics/http_clients/base.rb +15 -0
- data/lib/elastics/http_clients/loader.rb +51 -0
- data/lib/elastics/http_clients/patron.rb +29 -0
- data/lib/elastics/http_clients/rest_client.rb +36 -0
- data/lib/elastics/logger.rb +37 -0
- data/lib/elastics/prog_bar.rb +39 -0
- data/lib/elastics/rails.rb +1 -0
- data/lib/elastics/result.rb +24 -0
- data/lib/elastics/result/bulk.rb +20 -0
- data/lib/elastics/result/document.rb +67 -0
- data/lib/elastics/result/multi_get.rb +24 -0
- data/lib/elastics/result/search.rb +28 -0
- data/lib/elastics/struct/array.rb +25 -0
- data/lib/elastics/struct/hash.rb +105 -0
- data/lib/elastics/struct/paginable.rb +58 -0
- data/lib/elastics/struct/prunable.rb +60 -0
- data/lib/elastics/struct/symbolize.rb +27 -0
- data/lib/elastics/tasks.rb +62 -0
- data/lib/elastics/template.rb +124 -0
- data/lib/elastics/template/common.rb +42 -0
- data/lib/elastics/template/logger.rb +66 -0
- data/lib/elastics/template/partial.rb +28 -0
- data/lib/elastics/template/search.rb +30 -0
- data/lib/elastics/template/slim_search.rb +13 -0
- data/lib/elastics/template/tags.rb +56 -0
- data/lib/elastics/templates.rb +20 -0
- data/lib/elastics/utility_methods.rb +131 -0
- data/lib/elastics/utils.rb +103 -0
- data/lib/elastics/variables.rb +62 -0
- data/lib/tasks.rake +28 -0
- metadata +148 -0
@@ -0,0 +1,29 @@
|
|
1
|
+
module Elastics
|
2
|
+
module ClassProxy
|
3
|
+
class Base
|
4
|
+
|
5
|
+
attr_accessor :variables
|
6
|
+
|
7
|
+
def initialize(context, vars={})
|
8
|
+
@variables = Vars.new({:context => context,
|
9
|
+
:index => Conf.variables[:index]}.merge(vars))
|
10
|
+
end
|
11
|
+
|
12
|
+
def init; end
|
13
|
+
|
14
|
+
[:context, :index, :type].each do |meth|
|
15
|
+
define_method meth do
|
16
|
+
variables[meth]
|
17
|
+
end
|
18
|
+
define_method :"#{meth}=" do |val|
|
19
|
+
variables[meth] = val
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def refresh_index
|
24
|
+
Elastics.refresh_index :index => index
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
module Elastics
|
2
|
+
module ClassProxy
|
3
|
+
module Templates
|
4
|
+
|
5
|
+
attr_reader :templates, :partials
|
6
|
+
include Doc
|
7
|
+
include Search
|
8
|
+
|
9
|
+
def init
|
10
|
+
@sources = []
|
11
|
+
@templates = {}
|
12
|
+
@partials = {}
|
13
|
+
end
|
14
|
+
|
15
|
+
# accepts a path to a file or YAML string
|
16
|
+
def load_source_for(klass, source, source_vars)
|
17
|
+
if source.nil? || source !~ /\n/m
|
18
|
+
paths = [ "#{Conf.elastics_dir}/#{source}.yml",
|
19
|
+
"#{Conf.elastics_dir}/#{Utils.class_name_to_type(context.name)}.yml",
|
20
|
+
source.to_s ]
|
21
|
+
source = paths.find {|p| File.exist?(p)}
|
22
|
+
end
|
23
|
+
raise ArgumentError, "expected a string, got #{source.inspect}" \
|
24
|
+
unless source.is_a?(String)
|
25
|
+
@sources << [klass, source, source_vars]
|
26
|
+
do_load_source(klass, source, source_vars)
|
27
|
+
# fixes the legacy empty stubs, which should call super instead
|
28
|
+
@templates.keys.each do |name|
|
29
|
+
meta_context.send(:define_method, name){|*vars| super *vars }
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# loads a Generic Template source
|
34
|
+
def load_source(source=nil, source_vars=nil)
|
35
|
+
load_source_for(Template, source, source_vars)
|
36
|
+
end
|
37
|
+
|
38
|
+
# loads a Search Template source
|
39
|
+
def load_search_source(source=nil, source_vars=nil)
|
40
|
+
load_source_for(Template::Search, source, source_vars)
|
41
|
+
end
|
42
|
+
|
43
|
+
# loads a SlimSearch Template source
|
44
|
+
def load_slim_search_source(source=nil, source_vars=nil)
|
45
|
+
load_source_for(Template::SlimSearch, source, source_vars)
|
46
|
+
end
|
47
|
+
|
48
|
+
# reloads the sources (useful in the console and used internally)
|
49
|
+
def reload!
|
50
|
+
@sources.each {|k,s,v| do_load_source(k,s,v)}
|
51
|
+
end
|
52
|
+
|
53
|
+
def wrap(*methods, &block)
|
54
|
+
methods = templates.keys if methods.empty?
|
55
|
+
methods.each do |name|
|
56
|
+
raise MissingTemplateMethodError, "#{name} is not a template method" \
|
57
|
+
unless templates.include?(name)
|
58
|
+
meta_context.send(:define_method, name, &block)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def render(name, *vars)
|
63
|
+
templates[name].render(*vars)
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
def do_load_source(klass, source, source_vars)
|
69
|
+
source = Utils.erb_process(source) unless source.match("\n") # skips non-path
|
70
|
+
hash = Utils.parse_source(source)
|
71
|
+
Utils.delete_allcaps_keys(hash).each do |name, args|
|
72
|
+
define_template klass, name, args, source_vars
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def define_template(klass, name, args, source_vars)
|
77
|
+
name = name.to_sym
|
78
|
+
args = [args] unless args.is_a?(Array)
|
79
|
+
if name.to_s[0] == '_' # partial
|
80
|
+
partials[name] = Template::Partial.new(*args).setup(self, name)
|
81
|
+
else
|
82
|
+
templates[name] = klass.new(*args).setup(self, name, source_vars)
|
83
|
+
context::ElasticsTemplateMethods.send(:define_method, name) do |*vars|
|
84
|
+
raise ArgumentError, "#{elastics.context}.#{name} expects a list of Hashes, got #{vars.map(&:inspect).join(', ')}" \
|
85
|
+
unless vars.all?{|i| i.nil? || i.is_a?(Hash)}
|
86
|
+
elastics.render(name,*vars)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def meta_context
|
92
|
+
class << context; self end
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
module Elastics
|
2
|
+
module ClassProxy
|
3
|
+
module Templates
|
4
|
+
module Doc
|
5
|
+
|
6
|
+
def doc(*names)
|
7
|
+
names = templates.keys if names.empty?
|
8
|
+
doc = "\n"
|
9
|
+
names.each do |name|
|
10
|
+
next unless templates.include?(name)
|
11
|
+
block = ''
|
12
|
+
temp = templates[name]
|
13
|
+
meth_call = [context, name].join('.')
|
14
|
+
block << <<-meth_call
|
15
|
+
########## #{meth_call} ##########
|
16
|
+
#{'-' * temp.class.to_s.length}
|
17
|
+
#{temp.class}
|
18
|
+
#{temp.to_source}
|
19
|
+
meth_call
|
20
|
+
temp.partials.each do |par_name|
|
21
|
+
par = partials[par_name]
|
22
|
+
block << <<-partial
|
23
|
+
#{'-' * par.class.to_s.length}
|
24
|
+
#{par.class}
|
25
|
+
#{par.to_source}
|
26
|
+
partial
|
27
|
+
end
|
28
|
+
block << "\nUsage:\n"
|
29
|
+
block << build_usage(meth_call, temp)
|
30
|
+
block << "\n "
|
31
|
+
doc << block.split("\n").map{|l| '# ' + l}.join("\n")
|
32
|
+
doc << <<-meth.gsub(/^ {14}/m,'')
|
33
|
+
|
34
|
+
def #{meth_call}(*vars)
|
35
|
+
## this is a stub, used for reference
|
36
|
+
super
|
37
|
+
end
|
38
|
+
|
39
|
+
meth
|
40
|
+
end
|
41
|
+
puts doc
|
42
|
+
end
|
43
|
+
|
44
|
+
def usage(name)
|
45
|
+
meth_call = [context, name].join('.')
|
46
|
+
puts build_usage(meth_call, templates[name])
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def build_usage(meth_call, temp)
|
52
|
+
variables = temp.instance_eval do
|
53
|
+
interpolate
|
54
|
+
@base_variables.deep_merge @host_elastics && @host_elastics.variables, @temp_variables
|
55
|
+
end
|
56
|
+
all_tags = temp.tags + temp.partials
|
57
|
+
return meth_call if all_tags.size == 0
|
58
|
+
lines = all_tags.map do |t|
|
59
|
+
comments = 'partial' if t.to_s[0] == '_'
|
60
|
+
line = ['', t.inspect]
|
61
|
+
line + if variables.has_key?(t)
|
62
|
+
["#{variables[t].inspect},", comments_to_s(comments)]
|
63
|
+
else
|
64
|
+
["#{to_code(t)},", comments_to_s(comments, 'required')]
|
65
|
+
end
|
66
|
+
end
|
67
|
+
lines.sort! { |a,b| b[3] <=> a[3] }
|
68
|
+
lines.first[0] = meth_call
|
69
|
+
lines.last[2].chop!
|
70
|
+
max = lines.transpose.map { |c| c.map(&:length).max }
|
71
|
+
lines.map { |line| "%-#{max[0]}s %-#{max[1]}s => %-#{max[2]}s %s" % line }.join("\n")
|
72
|
+
end
|
73
|
+
|
74
|
+
def comments_to_s(*comments)
|
75
|
+
comments = comments.compact
|
76
|
+
return '' if comments == []
|
77
|
+
"# #{comments.join(' ')}"
|
78
|
+
end
|
79
|
+
|
80
|
+
def to_code(name)
|
81
|
+
keys = name.to_s.split('.').map{|s| s =~ /^[0..9]+$/ ? s.to_i : s.to_sym}
|
82
|
+
code = keys.shift.to_s
|
83
|
+
return code if keys.empty?
|
84
|
+
keys.each{|k| code << "[#{k.inspect}]"}
|
85
|
+
code
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
module Elastics
|
2
|
+
module ClassProxy
|
3
|
+
module Templates
|
4
|
+
module Search
|
5
|
+
|
6
|
+
def define_search(name, source, source_vars=nil)
|
7
|
+
args = Utils.parse_source(source)
|
8
|
+
send :define_template, Template::Search, name, args, source_vars
|
9
|
+
end
|
10
|
+
|
11
|
+
# http://www.elasticsearch.org/guide/reference/api/multi-search.html
|
12
|
+
# requests can be an array of arrays: [[:template1, variable_hash1], [template2, variable_hash2]]
|
13
|
+
# or a hash {:template1 => variable_hash1, template2 => variable_hash2}
|
14
|
+
# The variables are an hash of variables that will be used to render the msearch template
|
15
|
+
# the array of result is at <result>.responses
|
16
|
+
def multi_search(requests, *variables)
|
17
|
+
requests = requests.map { |name, vars| [name, vars] } if requests.is_a?(Hash)
|
18
|
+
lines = requests.map { |name, vars| templates[name].to_msearch(vars) }.join()
|
19
|
+
template = Elastics::Template.new('GET', '/<<index>>/<<type>>/_msearch') # no setup elastics so raw_result
|
20
|
+
template.send(:do_render, *variables, :data => lines) do |http_response|
|
21
|
+
responses = []
|
22
|
+
es_response = MultiJson.decode(http_response.body)
|
23
|
+
es_response['responses'].each_with_index do |raw_result, i|
|
24
|
+
name, vars = requests[i]
|
25
|
+
int = templates[name].interpolate(vars, strict=true)
|
26
|
+
result = Result.new(templates[name], int[:vars], http_response, raw_result)
|
27
|
+
responses << result.to_elastics_result
|
28
|
+
end
|
29
|
+
es_response['responses'] = responses
|
30
|
+
def es_response.responses
|
31
|
+
self['responses']
|
32
|
+
end
|
33
|
+
es_response
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# implements search_type=scan (http://www.elasticsearch.org/guide/reference/api/search/search-type.html)
|
38
|
+
def scan_search(template, *vars, &block)
|
39
|
+
user_raw_result = vars.any?{|v| v[:raw_result]}
|
40
|
+
scroll = '5m'
|
41
|
+
search_vars = Vars.new({:params => { :search_type => 'scan',
|
42
|
+
:scroll => scroll,
|
43
|
+
:size => 50 },
|
44
|
+
:raw_result => true}, *vars)
|
45
|
+
search_temp = template.is_a?(Elastics::Template) ? template : templates[template]
|
46
|
+
|
47
|
+
scroll_vars = Vars.new({:params => { :scroll => scroll },
|
48
|
+
:raw_result => true}, variables, *vars)
|
49
|
+
scroll_temp = Elastics::Template.new( :get,
|
50
|
+
'/_search/scroll',
|
51
|
+
nil,
|
52
|
+
scroll_vars )
|
53
|
+
search_res = search_temp.render search_vars
|
54
|
+
scroll_id = search_res['_scroll_id']
|
55
|
+
while (result = scroll_temp.render(:data => scroll_id)) do
|
56
|
+
break if result['hits']['hits'].empty?
|
57
|
+
scroll_id = result['_scroll_id']
|
58
|
+
result.variables[:raw_result] = user_raw_result
|
59
|
+
block.call result.to_elastics_result(force=true)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# implements search_type=count (http://www.elasticsearch.org/guide/reference/api/search/search-type.html)
|
64
|
+
def count_search(template, *vars)
|
65
|
+
template = template.is_a?(Elastics::Template) ? template : templates[template]
|
66
|
+
template.render Vars.new({:params => {:search_type => 'count'}, :raw_result => true}, *vars)
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Elastics
|
2
|
+
|
3
|
+
Configuration = OpenStruct.new :result_extenders => [ Result::Document,
|
4
|
+
Result::Search,
|
5
|
+
Result::MultiGet,
|
6
|
+
Result::Bulk ],
|
7
|
+
:logger => Logger.new(STDERR),
|
8
|
+
:variables => Vars.new( :index => nil,
|
9
|
+
:type => nil,
|
10
|
+
:params => {},
|
11
|
+
:no_pruning => [] ),
|
12
|
+
:config_file => './config/elastics.yml',
|
13
|
+
:elastics_dir => './elastics',
|
14
|
+
:http_client => HttpClients::Loader.new_http_client
|
15
|
+
|
16
|
+
# shorter alias
|
17
|
+
Conf = Configuration
|
18
|
+
|
19
|
+
Conf.instance_eval do
|
20
|
+
def configure
|
21
|
+
yield self
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
@@ -0,0 +1,153 @@
|
|
1
|
+
module Elastics
|
2
|
+
|
3
|
+
module Deprecation
|
4
|
+
|
5
|
+
extend self
|
6
|
+
|
7
|
+
def warn(old, new, called=1)
|
8
|
+
message = "#{old} is deprecated in favour of #{new}, and will be removed in a next version. Please, read the upgrade notes at http://elastics.github.io/elastics/doc/7-Tutorials/2-Migrate-from-0.x.html. "
|
9
|
+
message << "(called at: #{caller[called]})" if called
|
10
|
+
Conf.logger.warn message
|
11
|
+
end
|
12
|
+
|
13
|
+
module Module
|
14
|
+
def extended(obj)
|
15
|
+
Deprecation.warn(self, self::NEW_MODULE, 2)
|
16
|
+
obj.extend self::NEW_MODULE
|
17
|
+
end
|
18
|
+
def included(base)
|
19
|
+
Deprecation.warn(self, self::NEW_MODULE, 2)
|
20
|
+
base.send :include, self::NEW_MODULE
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
|
26
|
+
### Deprecation of Flex methods ###
|
27
|
+
|
28
|
+
def info(*names)
|
29
|
+
Deprecation.warn 'Flex.info', 'Elastics.doc'
|
30
|
+
doc *names
|
31
|
+
end
|
32
|
+
|
33
|
+
def process_bulk(options={})
|
34
|
+
Deprecation.warn 'Flex.process_bulk(:collection => collection)', 'Elastics.post_bulk_collection(collection, options)'
|
35
|
+
post_bulk_collection(options.delete(:collection), options)
|
36
|
+
end
|
37
|
+
|
38
|
+
def import_collection(collection, options={})
|
39
|
+
Deprecation.warn 'Flex.import_collection', 'Elastics.post_bulk_collection'
|
40
|
+
post_bulk_collection(collection, options.merge(:action => 'index'))
|
41
|
+
end
|
42
|
+
|
43
|
+
def delete_collection(collection, options={})
|
44
|
+
Deprecation.warn 'Flex.delete_collection(collection)', 'Elastics.post_bulk_collection(collection, :action => "delete")'
|
45
|
+
post_bulk_collection(collection, options.merge(:action => 'delete'))
|
46
|
+
end
|
47
|
+
|
48
|
+
def bulk(*vars)
|
49
|
+
Deprecation.warn 'Flex.bulk(:lines => lines_bulk_string)', 'Elastics.post_bulk_string(:bulk_string => lines_bulk_string)'
|
50
|
+
vars = Vars.new(*vars)
|
51
|
+
post_bulk_string(:bulk_string => vars[:lines])
|
52
|
+
end
|
53
|
+
|
54
|
+
|
55
|
+
### Deprecation of Configuration methods ###
|
56
|
+
|
57
|
+
Configuration.instance_eval do
|
58
|
+
# temprary deprecation warnings
|
59
|
+
def base_uri
|
60
|
+
Deprecation.warn 'Flex::Configuration.base_uri', 'Elastics::Configuration.http_client.base_uri'
|
61
|
+
http_client.base_uri
|
62
|
+
end
|
63
|
+
def base_uri=(val)
|
64
|
+
Deprecation.warn 'Flex::Configuration.base_uri=', 'Elastics::Configuration.http_client.base_uri='
|
65
|
+
http_client.base_uri = val
|
66
|
+
end
|
67
|
+
def http_client_options
|
68
|
+
Deprecation.warn 'Flex::Configuration.http_client_options', 'Elastics::Configuration.http_client.options'
|
69
|
+
http_client.options
|
70
|
+
end
|
71
|
+
def http_client_options=(val)
|
72
|
+
Deprecation.warn 'Flex::Configuration.http_client_options=', 'Elastics::Configuration.http_client.options='
|
73
|
+
http_client.options = val
|
74
|
+
end
|
75
|
+
def raise_proc
|
76
|
+
Deprecation.warn 'Flex::Configuration.raise_proc', 'Elastics::Configuration.http_client.raise_proc'
|
77
|
+
http_client.raise_proc
|
78
|
+
end
|
79
|
+
def raise_proc=(val)
|
80
|
+
Deprecation.warn 'Flex::Configuration.raise_proc=', 'Elastics::Configuration.http_client.raise_proc='
|
81
|
+
http_client.raise_proc = val
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
|
86
|
+
class Variables
|
87
|
+
def add(*hashes)
|
88
|
+
Deprecation.warn 'Flex::Variables#add', 'Elastics::Variables#deep_merge!'
|
89
|
+
replace deep_merge(*hashes)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
module Struct::Mergeable
|
95
|
+
def add(*hashes)
|
96
|
+
Deprecation.warn 'Flex::Variables#add', 'Variables#deep_merge!'
|
97
|
+
replace deep_merge(*hashes)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
|
102
|
+
module Loader
|
103
|
+
NEW_MODULE = Templates
|
104
|
+
extend Deprecation::Module
|
105
|
+
end
|
106
|
+
|
107
|
+
|
108
|
+
module ClassProxy::Loader
|
109
|
+
NEW_MODULE = ClassProxy::Templates
|
110
|
+
extend Deprecation::Module
|
111
|
+
end
|
112
|
+
|
113
|
+
|
114
|
+
module ClassProxy::Templates
|
115
|
+
module Doc
|
116
|
+
def info(*names)
|
117
|
+
Deprecation.warn 'flex.info', 'elastics.doc'
|
118
|
+
doc *names
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
|
124
|
+
module Result::Collection
|
125
|
+
NEW_MODULE = Struct::Paginable
|
126
|
+
extend Deprecation::Module
|
127
|
+
end
|
128
|
+
|
129
|
+
|
130
|
+
module Model
|
131
|
+
def self.included(base)
|
132
|
+
if defined?(Flex::ModelIndexer)
|
133
|
+
Deprecation.warn 'Flex::Model', 'Elastics::ModelIndexer'
|
134
|
+
base.send :include, Elastics::ModelIndexer
|
135
|
+
else
|
136
|
+
raise NotImplementedError, %(Elastics does not include "Flex::Model" anymore. Please, require the "elastics-models" gem, and include "Elastics::ModelIndexer" instead.)
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
|
142
|
+
module RelatedModel
|
143
|
+
def self.included(base)
|
144
|
+
if defined?(Flex::ModelSyncer)
|
145
|
+
Deprecation.warn 'Flex::RelatedModel', 'Elastics::ModelSyncer'
|
146
|
+
base.send :include, Elastics::ModelSyncer
|
147
|
+
else
|
148
|
+
raise NotImplementedError, %(Elastics does not include "Flex::RelatedModel" anymore. Please, require the "elastics-models" gem, and include "Elastics::ModelSyncer" instead.)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
end
|