elastic_record 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.travis.yml ADDED
@@ -0,0 +1,4 @@
1
+ services:
2
+ - elasticsearch
3
+ rvm:
4
+ - 1.9.3
data/Gemfile CHANGED
@@ -1,4 +1,3 @@
1
1
  source :rubygems
2
-
3
2
  gemspec
4
- # gem 'arelastic', path: '~/code/arelastic'
3
+ gem 'rake'
data/README.rdoc CHANGED
@@ -1,4 +1,5 @@
1
1
  = ElasticRecord
2
+ {<img src="https://secure.travis-ci.org/matthuhiggins/elastic_record.png?rvm=1.9.3" />}[http://travis-ci.org/matthuhiggins/elastic_record]
2
3
 
3
4
  ElasticRecord is an elasticsearch ORM.
4
5
 
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |s|
4
4
  s.name = 'elastic_record'
5
- s.version = '0.5.0'
5
+ s.version = '0.6.0'
6
6
  s.summary = 'Use Elastic Search with your objects'
7
7
  s.description = 'Find your records with elastic search'
8
8
 
@@ -19,6 +19,5 @@ Gem::Specification.new do |s|
19
19
  s.test_files = `git ls-files -- {test}/*`.split("\n")
20
20
 
21
21
  s.add_dependency 'arelastic'
22
- s.add_dependency 'rubberband', '>= 0.1.1'
23
22
  s.add_dependency 'activemodel'
24
23
  end
@@ -3,11 +3,11 @@ module ElasticRecord
3
3
  def self.included(base)
4
4
  base.class_eval do
5
5
  after_save do
6
- self.class.elastic_index.index_record self
6
+ self.class.elastic_index.index_document id, as_search
7
7
  end
8
8
 
9
9
  after_destroy do
10
- self.class.elastic_index.delete_record self
10
+ self.class.elastic_index.delete_document id
11
11
  end
12
12
  end
13
13
  end
@@ -1,14 +1,70 @@
1
1
  module ElasticRecord
2
- module Connection
3
- def elastic_connection
4
- @elastic_connection ||= ElasticSearch.new(
5
- ElasticRecord::Config.servers,
6
- ElasticRecord::Config.connection_options.merge(index: elastic_index.alias_name, type: elastic_index.type)
7
- )
2
+ class Connection
3
+ # :timeout: 10
4
+ # :retries: 2
5
+ # :auto_discovery: false
6
+
7
+ attr_accessor :servers, :options
8
+ def initialize(servers, options = {})
9
+ if servers.is_a?(Array)
10
+ self.servers = servers
11
+ else
12
+ self.servers = servers.split(',')
13
+ end
14
+
15
+ self.options = options
16
+ end
17
+
18
+ def head(path)
19
+ http_request(Net::HTTP::Head, path).code
20
+ end
21
+
22
+ def json_get(path, json = nil)
23
+ json_request Net::HTTP::Get, path, json
24
+ end
25
+
26
+ def json_post(path, json = nil)
27
+ json_request Net::HTTP::Post, path, json
28
+ end
29
+
30
+ def json_put(path, json = nil)
31
+ json_request Net::HTTP::Put, path, json
8
32
  end
9
33
 
10
- def elastic_connection=(connection)
11
- @elastic_connection = connection
34
+ def json_delete(path, json = nil)
35
+ json_request Net::HTTP::Delete, path, json
12
36
  end
37
+
38
+ def json_request(request_klass, path, json)
39
+ body = json.is_a?(Hash) ? ActiveSupport::JSON.encode(json) : json
40
+ response = http_request(request_klass, path, body)
41
+ json = ActiveSupport::JSON.decode response.body
42
+
43
+ raise json['error'] if json['error']
44
+
45
+ json
46
+ end
47
+
48
+ def http_request(request_klass, path, body = nil)
49
+ request = request_klass.new(path)
50
+ request.body = body
51
+
52
+ http.request(request)
53
+ end
54
+
55
+ private
56
+ def choose_server
57
+ servers.sample
58
+ end
59
+
60
+ def http
61
+ host, port = choose_server.split ':'
62
+
63
+ http = Net::HTTP.new(host, port)
64
+ if options[:timeout]
65
+ http.read_timeout = options[:timeout].to_i
66
+ end
67
+ http
68
+ end
13
69
  end
14
70
  end
@@ -1,25 +1,47 @@
1
+ require 'active_support/core_ext/object/to_query'
2
+
1
3
  module ElasticRecord
2
4
  class Index
3
5
  module Documents
4
- def index_record(record, index_name = nil)
6
+ def index_document(id, document, index_name = nil)
5
7
  return if disabled
6
8
 
7
9
  index_name ||= alias_name
8
- document = record.respond_to?(:as_search) ? record.as_search : {}
9
10
 
10
- connection.index(document, id: record.id, index: index_name)
11
- # json_put "#{index_name}/#{type}/#{record.id}", document
11
+ if @batch
12
+ @batch << { index: { _index: index_name, _type: type, _id: id } }
13
+ @batch << document
14
+ else
15
+ connection.json_put "/#{index_name}/#{type}/#{id}", document
16
+ end
12
17
  end
13
18
 
14
- def delete_record(record, index_name = nil)
19
+ def delete_document(id, index_name = nil)
15
20
  index_name ||= alias_name
16
21
 
17
- connection.delete(record.id, index: index_name)
18
- # json_delete "#{index_name}/#{type}/#{record.id}"
22
+ if @batch
23
+ @batch << { delete: { _index: index_name, _type: type, _id: id } }
24
+ else
25
+ connection.json_delete "/#{index_name}/#{type}/#{id}"
26
+ end
19
27
  end
20
28
 
21
29
  def record_exists?(id)
22
- !connection.get(id).nil?
30
+ connection.json_get("/#{alias_name}/#{type}/#{id}")['exists']
31
+ end
32
+
33
+ def search(elastic_query, options = {})
34
+ url = "/#{alias_name}/#{type}/_search"
35
+ if options.any?
36
+ url += "?#{options.to_query}"
37
+ end
38
+
39
+ connection.json_get url, elastic_query
40
+ end
41
+
42
+ def scroll(scroll_id, scroll_keep_alive)
43
+ options = {scroll_id: scroll_id, scroll: scroll_keep_alive}
44
+ connection.json_get("/_search/scroll?#{options.to_query}")
23
45
  end
24
46
 
25
47
  def bulk_add(batch, index_name = nil)
@@ -27,12 +49,23 @@ module ElasticRecord
27
49
 
28
50
  index_name ||= alias_name
29
51
 
30
- connection.bulk do
52
+ bulk do
31
53
  batch.each do |record|
32
- index_record(record, index_name)
54
+ index_document(record.id, record.as_search, index_name)
33
55
  end
34
56
  end
35
57
  end
58
+
59
+ def bulk
60
+ @batch = []
61
+ yield
62
+ if @batch.any?
63
+ body = @batch.map { |action| "#{ActiveSupport::JSON.encode(action)}\n" }.join
64
+ connection.json_post "/_bulk", body
65
+ end
66
+ ensure
67
+ @batch = nil
68
+ end
36
69
  end
37
70
  end
38
71
  end
@@ -8,13 +8,13 @@ module ElasticRecord
8
8
  end
9
9
 
10
10
  def create(index_name = new_index_name)
11
- json_put "/#{index_name}"
11
+ connection.json_put "/#{index_name}"
12
12
  update_mapping(index_name)
13
13
  index_name
14
14
  end
15
15
 
16
16
  def delete(index_name)
17
- json_delete "/#{index_name}"
17
+ connection.json_delete "/#{index_name}"
18
18
  end
19
19
 
20
20
  def delete_all
@@ -24,7 +24,7 @@ module ElasticRecord
24
24
  end
25
25
 
26
26
  def exists?(index_name)
27
- head("/#{index_name}") == '200'
27
+ connection.head("/#{index_name}") == '200'
28
28
  end
29
29
 
30
30
  def deploy(index_name)
@@ -46,15 +46,15 @@ module ElasticRecord
46
46
  }
47
47
  end
48
48
 
49
- json_post '/_aliases', actions: actions
49
+ connection.json_post '/_aliases', actions: actions
50
50
  end
51
51
 
52
52
  def update_mapping(index_name)
53
- json_put "/#{index_name}/#{type}/_mapping", type => mapping
53
+ connection.json_put "/#{index_name}/#{type}/_mapping", type => mapping
54
54
  end
55
55
 
56
56
  def refresh
57
- connection.refresh
57
+ connection.json_post "/#{alias_name}/_refresh"
58
58
  end
59
59
 
60
60
  def reset
@@ -63,12 +63,12 @@ module ElasticRecord
63
63
  end
64
64
 
65
65
  def aliased_names
66
- json = json_get '/_cluster/state'
66
+ json = connection.json_get '/_cluster/state'
67
67
  json["metadata"]["indices"].select { |name, status| status["aliases"].include?(alias_name) }.map { |name, status| name }
68
68
  end
69
69
 
70
70
  def all_names
71
- json = json_get '/_status'
71
+ json = connection.json_get '/_status'
72
72
 
73
73
  regex = %r{^#{alias_name}_?}
74
74
  json['indices'].keys.grep(regex)
@@ -8,19 +8,19 @@ module ElasticRecord
8
8
  update_mapping percolator_index_name
9
9
  end
10
10
 
11
- json_put "/_percolator/#{percolator_index_name}/#{name}", elastic_query
11
+ connection.json_put "/_percolator/#{percolator_index_name}/#{name}", elastic_query
12
12
  end
13
13
 
14
14
  def delete_percolator(name)
15
- json_delete "/_percolator/#{percolator_index_name}/#{name}"
15
+ connection.json_delete "/_percolator/#{percolator_index_name}/#{name}"
16
16
  end
17
17
 
18
18
  def percolator_exists?(name)
19
- json_get("/_percolator/#{percolator_index_name}/#{name}")['exists']
19
+ connection.json_get("/_percolator/#{percolator_index_name}/#{name}")['exists']
20
20
  end
21
21
 
22
22
  def percolate(document)
23
- json_get("/#{percolator_index_name}/#{type}/_percolate", 'doc' => document)['matches']
23
+ connection.json_get("/#{percolator_index_name}/#{type}/_percolate", 'doc' => document)['matches']
24
24
  end
25
25
 
26
26
  def percolator_index_name
@@ -35,55 +35,13 @@ module ElasticRecord
35
35
  @disabled = false
36
36
  end
37
37
 
38
- # private
38
+ private
39
39
  def new_index_name
40
40
  "#{alias_name}_#{Time.now.to_i}"
41
41
  end
42
42
 
43
- def head(path)
44
- http_request(Net::HTTP::Head, path).code
45
- end
46
-
47
- def json_get(path, json = nil)
48
- json_request Net::HTTP::Get, path, json
49
- end
50
-
51
- def json_post(path, json = nil)
52
- json_request Net::HTTP::Post, path, json
53
- end
54
-
55
- def json_put(path, json = nil)
56
- json_request Net::HTTP::Put, path, json
57
- end
58
-
59
- def json_delete(path, json = nil)
60
- json_request Net::HTTP::Delete, path, json
61
- end
62
-
63
- def json_request(request_klass, path, json)
64
- body = json ? ActiveSupport::JSON.encode(json) : nil
65
- response = http_request(request_klass, path, body)
66
- json = ActiveSupport::JSON.decode response.body
67
-
68
- raise json['error'] if json['error']
69
-
70
- json
71
- end
72
-
73
- def http_request(request_klass, path, body = nil)
74
- request = request_klass.new(path)
75
- request.body = body
76
-
77
- http.request(request)
78
- end
79
-
80
43
  def connection
81
- @model.elastic_connection
82
- end
83
-
84
- def http
85
- host, port = connection.current_server.split ':'
86
- Net::HTTP.new(host, port)
44
+ model.elastic_connection
87
45
  end
88
46
  end
89
47
  end
@@ -2,11 +2,19 @@ module ElasticRecord
2
2
  module Model
3
3
  def self.included(base)
4
4
  base.class_eval do
5
- extend Connection, Searching, ClassMethods
5
+ extend Searching, ClassMethods
6
6
  end
7
7
  end
8
8
 
9
9
  module ClassMethods
10
+ def elastic_connection
11
+ @elastic_connection ||= ElasticRecord::Connection.new(ElasticRecord::Config.servers)
12
+ end
13
+
14
+ def elastic_connection=(connection)
15
+ @elastic_connection = connection
16
+ end
17
+
10
18
  def elastic_relation
11
19
  ElasticRecord::Relation.new(self, arelastic)
12
20
  end
@@ -8,14 +8,18 @@ module ElasticRecord
8
8
  end
9
9
 
10
10
  def find_in_batches(options = {})
11
- options[:size] ||= 100
12
- options[:scroll] ||= '20m'
13
- options[:search_type] = 'scan'
11
+ scroll_keep_alive = '10m'
14
12
 
15
- hits = klass.elastic_connection.search(as_elastic, options)
13
+ options = {
14
+ scroll: scroll_keep_alive,
15
+ size: 100,
16
+ search_type: 'scan'
17
+ }
16
18
 
17
- klass.elastic_connection.scroll(hits.scroll_id, scroll: options[:scroll], ids_only: true) do |hits|
18
- yield klass.find(hits.to_a)
19
+ scroll_id = klass.elastic_index.search(as_elastic, options)['_scroll_id']
20
+
21
+ while (hit_ids = get_scroll_hit_ids(scroll_id, scroll_keep_alive)).any?
22
+ yield klass.find(hit_ids)
19
23
  end
20
24
  end
21
25
 
@@ -24,6 +28,12 @@ module ElasticRecord
24
28
  elastic_index.bulk_add(batch)
25
29
  end
26
30
  end
31
+
32
+ private
33
+ def get_scroll_hit_ids(scroll_id, scroll_keep_alive)
34
+ json = klass.elastic_index.scroll(scroll_id, scroll_keep_alive)
35
+ json['hits']['hits'].map { |hit| hit['_id'] }
36
+ end
27
37
  end
28
38
  end
29
- end
39
+ end
@@ -18,11 +18,11 @@ module ElasticRecord
18
18
  end
19
19
 
20
20
  def count
21
- to_hits.total_entries
21
+ search_results['hits']['total']
22
22
  end
23
23
 
24
24
  def facets
25
- to_hits.facets
25
+ search_results['facets']
26
26
  end
27
27
 
28
28
  def create_percolator(name)
@@ -30,7 +30,7 @@ module ElasticRecord
30
30
  end
31
31
 
32
32
  def reset
33
- @hits = @records = nil
33
+ @search_results = @records = nil
34
34
  end
35
35
 
36
36
  def initialize_copy(other)
@@ -46,11 +46,7 @@ module ElasticRecord
46
46
  end
47
47
 
48
48
  def to_ids
49
- to_hits.to_a.map(&:id)
50
- end
51
-
52
- def to_hits
53
- @hits ||= klass.elastic_connection.search(as_elastic)#, ids_only: true)
49
+ search_hits.map { |hit| hit['_id'] }
54
50
  end
55
51
 
56
52
  def ==(other)
@@ -72,5 +68,14 @@ module ElasticRecord
72
68
  ensure
73
69
  klass.current_elastic_search = previous
74
70
  end
71
+
72
+ private
73
+ def search_hits
74
+ search_results['hits']['hits']
75
+ end
76
+
77
+ def search_results
78
+ @search_results ||= klass.elastic_index.search(as_elastic)
79
+ end
75
80
  end
76
81
  end
@@ -1,5 +1,4 @@
1
1
  require 'arelastic'
2
- require 'rubberband'
3
2
  require 'active_support/core_ext/object/blank' # required because ActiveModel depends on this but does not require it
4
3
  require 'active_model'
5
4
 
@@ -17,7 +17,7 @@ class ElasticRecord::CallbacksTest < MiniTest::Spec
17
17
 
18
18
  def test_deleted_from_index
19
19
  widget = Widget.new id: '10', color: 'green'
20
- Widget.elastic_index.index_record(widget)
20
+ Widget.elastic_index.index_document(widget.id, widget.as_search)
21
21
 
22
22
  assert Widget.elastic_index.record_exists?(widget.id)
23
23
 
@@ -1,10 +1,8 @@
1
1
  require 'helper'
2
2
 
3
3
  class ElasticRecord::ConnectionTest < MiniTest::Spec
4
- def test_elastic_connection
5
- connection = Widget.elastic_connection
6
-
7
- assert_equal Widget.elastic_index.type, connection.default_type
8
- assert_equal Widget.elastic_index.alias_name, connection.default_index
4
+ def test_servers
5
+ assert_equal ['foo', 'bar'], ElasticRecord::Connection.new('foo,bar').servers
6
+ assert_equal ['foo', 'bar'], ElasticRecord::Connection.new(['foo', 'bar']).servers
9
7
  end
10
8
  end
@@ -7,22 +7,18 @@ class ElasticRecord::Index::DocumentsTest < MiniTest::Spec
7
7
  index.reset
8
8
  end
9
9
 
10
- def test_index_record
11
- record = Widget.new(id: 'abc', color: 'red')
12
-
13
- index.index_record(record)
10
+ def test_index_document
11
+ index.index_document('abc', color: 'red')
14
12
 
15
13
  assert index.record_exists?('abc')
16
14
  refute index.record_exists?('xyz')
17
15
  end
18
16
 
19
- def test_delete_record
20
- record = Widget.new(id: 'abc', color: 'red')
21
-
22
- index.index_record(record)
17
+ def test_delete_document
18
+ index.index_document('abc', color: 'red')
23
19
  assert index.record_exists?('abc')
24
20
 
25
- index.delete_record(record)
21
+ index.delete_document('abc')
26
22
  refute index.record_exists?('abc')
27
23
  end
28
24
 
@@ -35,6 +31,24 @@ class ElasticRecord::Index::DocumentsTest < MiniTest::Spec
35
31
  refute index.record_exists?('xyz')
36
32
  end
37
33
 
34
+ def test_bulk
35
+ assert_nil index.instance_variable_get(:@batch)
36
+
37
+ index.bulk do
38
+ index.index_document '5', color: 'green'
39
+ index.delete_document '3'
40
+
41
+ expected = [
42
+ {index: {_index: "widgets", _type: "widget", _id: "5"}},
43
+ {color: "green"},
44
+ {delete: {_index: "widgets", _type: "widget", _id: "3"}}
45
+ ]
46
+ assert_equal expected, index.instance_variable_get(:@batch)
47
+ end
48
+
49
+ assert_nil index.instance_variable_get(:@batch)
50
+ end
51
+
38
52
  private
39
53
  def index
40
54
  @index ||= Widget.elastic_index
@@ -1,6 +1,13 @@
1
1
  require 'helper'
2
2
 
3
3
  class ElasticRecord::ModelTest < MiniTest::Spec
4
+ # def test_elastic_connection
5
+ # connection = Widget.elastic_connection
6
+ #
7
+ # assert_equal Widget.elastic_index.type, connection.default_type
8
+ # assert_equal Widget.elastic_index.alias_name, connection.default_index
9
+ # end
10
+
4
11
  def test_elastic_relation
5
12
  relation = Widget.elastic_relation
6
13
 
@@ -6,7 +6,7 @@ class ElasticRecord::Relation::DelegationTest < MiniTest::Spec
6
6
  end
7
7
 
8
8
  def test_delegate_to_array
9
- Widget.elastic_index.index_record(Widget.new(id: 5, color: 'red'))
9
+ Widget.elastic_index.index_document('5', color: 'red')
10
10
  Widget.elastic_index.refresh
11
11
 
12
12
  records = []
@@ -159,8 +159,10 @@ class ElasticRecord::Relation::SearchMethodsTest < MiniTest::Spec
159
159
  end
160
160
 
161
161
  def test_select_with_block
162
- Widget.elastic_connection.index({'widget' => {'color' => 'red'}}, {index: 'widgets', type: 'widget', id: 5})
163
- Widget.elastic_connection.index({'widget' => {'color' => 'blue'}}, {index: 'widgets', type: 'widget', id: 10})
162
+ Widget.elastic_index.bulk_add [
163
+ Widget.new(id: 5, color: 'red'),
164
+ Widget.new(id: 10, color: 'blue')
165
+ ]
164
166
 
165
167
  records = relation.select { |record| record.id == '10' }
166
168
 
@@ -28,7 +28,7 @@ class ElasticRecord::RelationTest < MiniTest::Spec
28
28
  end
29
29
 
30
30
  def test_to_hits
31
- assert Widget.elastic_relation.to_hits.is_a?(ElasticSearch::Api::Hits)
31
+ # assert Widget.elastic_relation.search_results.is_a?(ElasticSearch::Api::Hits)
32
32
  end
33
33
 
34
34
  def test_to_ids
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elastic_record
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.0
4
+ version: 0.6.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-09-07 00:00:00.000000000 Z
12
+ date: 2012-09-10 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: arelastic
@@ -27,22 +27,6 @@ dependencies:
27
27
  - - ! '>='
28
28
  - !ruby/object:Gem::Version
29
29
  version: '0'
30
- - !ruby/object:Gem::Dependency
31
- name: rubberband
32
- requirement: !ruby/object:Gem::Requirement
33
- none: false
34
- requirements:
35
- - - ! '>='
36
- - !ruby/object:Gem::Version
37
- version: 0.1.1
38
- type: :runtime
39
- prerelease: false
40
- version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
- requirements:
43
- - - ! '>='
44
- - !ruby/object:Gem::Version
45
- version: 0.1.1
46
30
  - !ruby/object:Gem::Dependency
47
31
  name: activemodel
48
32
  requirement: !ruby/object:Gem::Requirement
@@ -67,6 +51,7 @@ extra_rdoc_files:
67
51
  - README.rdoc
68
52
  files:
69
53
  - .gitignore
54
+ - .travis.yml
70
55
  - Gemfile
71
56
  - MIT-LICENSE
72
57
  - README.rdoc