elastics-models 1.0.4

Sign up to get free protection for your applications and to get access to all the features.
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2012-2013 by Domizio Demichelis
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,32 @@
1
+ # elastics-models
2
+
3
+ [![Gem Version](https://badge.fury.io/rb/elastics-models.png)](http://badge.fury.io/rb/elastics-models)
4
+
5
+ Transparently integrates your models with one or more elasticsearch indices:
6
+
7
+ * Automatic integration with your `ActiveRecord` and `Mongoid` models
8
+ * Direct management of indices throught `ActiveModel`
9
+ * Validations and callbacks
10
+ * Typecasting
11
+ * Attribute defaults
12
+ * Persistent storage with optimistic lock update
13
+ * integration with the `elasticsearch-mapper-attachment` plugin
14
+ * finders, chainable scopes etc. {% see 4.3 %}
15
+ * Automatic generation of elasticsearch mappings based on your models
16
+ * Parent/Child Relationships
17
+ * Bulk import
18
+ * Real-time indexing and search capabilities
19
+
20
+ ## Links
21
+
22
+ - __Gem-Specific Documentation__
23
+ - [elastics-models](http://elastics.github.io/elastics/doc/4-elastics-models)
24
+
25
+ ## Credits
26
+
27
+ Special thanks for their sponsorship to [Escalate Media](http://www.escalatemedia.com) and [Barquin International](http://www.barquin.com).
28
+
29
+ ## Copyright
30
+
31
+ Copyright (c) 2012-2013 by [Domizio Demichelis](mailto://dd.nexus@gmail.com)<br>
32
+ See [LICENSE](https://github.com/elastics/elastics/blob/master/elastics-models/LICENSE) for details.
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 1.0.4
@@ -0,0 +1,22 @@
1
+ require 'date'
2
+ version = File.read(File.expand_path('../VERSION', __FILE__)).strip
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = 'elastics-models'
6
+ s.summary = 'Transparently integrates your models with one or more elasticsearch indices.'
7
+ s.description = 'Provides ActiveRecord, Mongoid, ActiveModel and elasticsearch-mapper-attachment integrations, cross syncing, parent/child relationships, bulk-import, live-reindex of models, ...'
8
+ s.homepage = 'http://elastics.github.io/elastics'
9
+ s.authors = ["Domizio Demichelis"]
10
+ s.email = 'dd.nexus@gmail.com'
11
+ s.files = `git ls-files -z`.split("\0")
12
+ s.version = version
13
+ s.date = Date.today.to_s
14
+ s.required_rubygems_version = ">= 1.3.6"
15
+ s.rdoc_options = %w[--charset=UTF-8]
16
+ s.license = 'MIT'
17
+
18
+ s.add_runtime_dependency 'elastics-client', version
19
+ s.add_runtime_dependency 'elastics-scopes', version
20
+
21
+ s.add_runtime_dependency 'active_attr', '>= 0.6.0'
22
+ end
@@ -0,0 +1,42 @@
1
+ require 'base64'
2
+ module Elastics
3
+ module ActiveModel
4
+ module Attachment
5
+
6
+ # defines accessors for <attachment_field_name>
7
+ # if you omit the arguments it uses :attachment as the <attachment_field_name>
8
+ # you can also pass other properties that will be merged with the default property for attachment
9
+ # this will automatically add a :<attachment_field_name>_scope scope which will add
10
+ # all the meta fields (title, author, ...) to the returned fields, exluding the <attachment_field_name> field itself
11
+ # and including all the other attributes declared before it. For that reason you may want to declare it as
12
+ # the latest attribute.
13
+
14
+ def attribute_attachment(*args)
15
+ name = args.first.is_a?(Symbol) ? args.shift : :attachment
16
+ props = {:properties => { 'type' => 'attachment',
17
+ 'fields' => { name.to_s => { 'store' => 'yes', 'term_vector' => 'with_positions_offsets' },
18
+ 'title' => { 'store' => 'yes' },
19
+ 'author' => { 'store' => 'yes' },
20
+ 'name' => { 'store' => 'yes' },
21
+ 'content_type' => { 'store' => 'yes' },
22
+ 'date' => { 'store' => 'yes' },
23
+ 'keywords' => { 'store' => 'yes' }
24
+ }
25
+ }
26
+ }
27
+ props.extend(Struct::Mergeable).deep_merge! args.first if args.first.is_a?(Hash)
28
+
29
+ scope :"#{name}_scope", fields("#{name}.title",
30
+ "#{name}.author",
31
+ "#{name}.name",
32
+ "#{name}.content_type",
33
+ "#{name}.date",
34
+ "#{name}.keywords",
35
+ *attributes.keys)
36
+ attribute name, props
37
+
38
+ end
39
+
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,21 @@
1
+ module Elastics
2
+ module ActiveModel
3
+ module Inspection
4
+
5
+ def inspect
6
+ descriptions = [%(_id: #{@_id.inspect}), %(_version: #{@_version})]
7
+ all_attributes = if respond_to?(:raw_document)
8
+ reader_keys = raw_document.send(:readers).keys.map(&:to_s)
9
+ # we send() the readers, so they will reflect an eventual overriding
10
+ Hash[ reader_keys.map{ |k| [k, send(k)] } ].merge(attributes)
11
+ else
12
+ attributes
13
+ end
14
+ descriptions << all_attributes.sort.map { |key, value| "#{key}: #{value.inspect}" }
15
+ separator = " " unless descriptions.empty?
16
+ "#<#{self.class.name}#{separator}#{descriptions.join(", ")}>"
17
+ end
18
+
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,127 @@
1
+ module Elastics
2
+ module ActiveModel
3
+
4
+ class DocumentInvalidError < StandardError; end
5
+
6
+ module Storage
7
+
8
+ module ClassMethods
9
+
10
+ def create(args={})
11
+ document = new(args)
12
+ return false unless document.valid?
13
+ document.save
14
+ end
15
+
16
+ end
17
+
18
+
19
+ module InstanceMethods
20
+
21
+ def reload
22
+ document = elastics.get
23
+ self.attributes = document['_source']
24
+ @_id = document['_id']
25
+ @_version = document['_version']
26
+ end
27
+
28
+ def save(options={})
29
+ perform_validations(options) ? do_save : false
30
+ end
31
+
32
+ def save!(options={})
33
+ perform_validations(options) ? do_save : raise(DocumentInvalidError, errors.full_messages.join(", "))
34
+ end
35
+
36
+ # Optimistic Lock Update
37
+ #
38
+ # doc.safe_update do |d|
39
+ # d.amount += 100
40
+ # end
41
+ #
42
+ # if you are trying to update a stale object, the block is yielded again with a fresh reloaded document and the
43
+ # document is saved only when it is not stale anymore (i.e. the _version has not changed since it has been loaded)
44
+ # read: http://www.elasticsearch.org/blog/2011/02/08/versioning.html
45
+ #
46
+ def safe_update(options={}, &block)
47
+ perform_validations(options) ? lock_update(&block) : false
48
+ end
49
+
50
+ def safe_update!(options={}, &block)
51
+ perform_validations(options) ? lock_update(&block) : raise(DocumentInvalidError, errors.full_messages.join(", "))
52
+ end
53
+
54
+ def valid?(context = nil)
55
+ context ||= (new_record? ? :create : :update)
56
+ output = super(context)
57
+ errors.empty? && output
58
+ end
59
+
60
+ def destroy
61
+ @destroyed = true
62
+ elastics.sync
63
+ self.freeze
64
+ end
65
+
66
+ def delete
67
+ @skip_destroy_callbacks = true
68
+ destroy
69
+ end
70
+
71
+ def merge_attributes(attributes)
72
+ attributes.each {|name, value| send "#{name}=", value }
73
+ end
74
+
75
+ def update_attributes(attributes)
76
+ merge_attributes(attributes)
77
+ save
78
+ end
79
+
80
+ def destroyed?
81
+ !!@destroyed
82
+ end
83
+
84
+ def persisted?
85
+ !(new_record? || destroyed?)
86
+ end
87
+
88
+ def new_record?
89
+ !@_id || !@_version
90
+ end
91
+
92
+ private
93
+
94
+ def do_save
95
+ elastics.sync
96
+ self
97
+ end
98
+
99
+ def lock_update
100
+ begin
101
+ yield self
102
+ elastics.sync
103
+ rescue Elastics::HttpError => e
104
+ if e.status == 409
105
+ reload
106
+ retry
107
+ else
108
+ raise
109
+ end
110
+ end
111
+ self
112
+ end
113
+
114
+ protected
115
+
116
+ def perform_validations(options={})
117
+ perform_validation = options[:validate] != false
118
+ perform_validation ? valid?(options[:context]) : true
119
+ end
120
+
121
+ end
122
+
123
+ end
124
+
125
+
126
+ end
127
+ end
@@ -0,0 +1,22 @@
1
+ module Elastics
2
+ module ActiveModel
3
+ module Timestamps
4
+
5
+ def attribute_timestamps(props={})
6
+ attribute_created_at props
7
+ attribute_updated_at props
8
+ end
9
+
10
+ def attribute_created_at(props={})
11
+ attribute :created_at, {:type => DateTime}.merge(props)
12
+ before_create { self.created_at = Time.now.utc }
13
+ end
14
+
15
+ def attribute_updated_at(props={})
16
+ attribute :updated_at, {:type => DateTime}.merge(props)
17
+ before_save { self.updated_at = Time.now.utc }
18
+ end
19
+
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,47 @@
1
+ module Elastics
2
+ module ActiveModel
3
+
4
+ attr_reader :_version, :_id, :highlight
5
+ alias_method :id, :_id
6
+
7
+ def self.included(base)
8
+ base.class_eval do
9
+ @elastics ||= ClassProxy::Base.new(base)
10
+ @elastics.extend(ClassProxy::ModelSyncer)
11
+ @elastics.extend(ClassProxy::ModelIndexer).init
12
+ @elastics.extend(ClassProxy::ActiveModel).init :params => {:version => true}
13
+ def self.elastics; @elastics end
14
+ elastics.synced = [self]
15
+
16
+ include Scopes
17
+ include ActiveAttr::Model
18
+
19
+ extend ::ActiveModel::Callbacks
20
+ define_model_callbacks :create, :update, :save, :destroy
21
+
22
+ include Storage::InstanceMethods
23
+ extend Storage::ClassMethods
24
+ include Inspection
25
+ extend Timestamps
26
+ extend Attachment
27
+ end
28
+ end
29
+
30
+ def elastics
31
+ @elastics ||= InstanceProxy::ActiveModel.new(self)
32
+ end
33
+
34
+ def elastics_source
35
+ attributes
36
+ end
37
+
38
+ def elastics_indexable?
39
+ true
40
+ end
41
+
42
+ def method_missing(meth, *args, &block)
43
+ raw_document.respond_to?(meth) ? raw_document.send(meth) : super
44
+ end
45
+
46
+ end
47
+ end
@@ -0,0 +1,35 @@
1
+ module Elastics
2
+ module ClassProxy
3
+ module ActiveModel
4
+
5
+ def init(*vars)
6
+ variables.deep_merge! *vars
7
+ end
8
+
9
+ def default_mapping
10
+ props = { }
11
+ context.attributes.each do |name, attr|
12
+ options = attr.send(:options)
13
+ props[name] = case
14
+ when options.has_key?(:properties)
15
+ Utils.keyfy(:to_s, attr.send(:options)[:properties])
16
+ when options.has_key?(:not_analyzed) && options[:not_analyzed] ||
17
+ options.has_key?(:analyzed) && !options[:analyzed]
18
+ { 'type' => 'string', 'index' => 'not_analyzed' }
19
+ when options[:type] == DateTime
20
+ { 'type' => 'date', 'format' => 'dateOptionalTime' }
21
+ else
22
+ next
23
+ end
24
+ end
25
+ props.empty? ? super : super.deep_merge(index => {'mappings' => {type => {'properties' => props}}})
26
+ end
27
+
28
+ # overrides the ModelSyncer#add_callbacks
29
+ def add_callbacks
30
+ # no callbacks to add, since it calls elastics.sync on save and destroy
31
+ end
32
+
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,46 @@
1
+ module Elastics
2
+ module ClassProxy
3
+ module ModelIndexer
4
+
5
+ module Types
6
+ extend self
7
+
8
+ attr_accessor :parents
9
+ @parents = []
10
+ end
11
+
12
+ attr_reader :parent_association, :parent_child_map
13
+
14
+ def init
15
+ variables.deep_merge! :type => Utils.class_name_to_type(context.name)
16
+ end
17
+
18
+ def parent(parent_association, map)
19
+ @parent_association = parent_association
20
+ Types.parents |= map.keys.map(&:to_s)
21
+ self.type = map.values.map(&:to_s)
22
+ @parent_child_map = map
23
+ @is_child = true
24
+ end
25
+
26
+ def is_child?
27
+ !!@is_child
28
+ end
29
+
30
+ def is_parent?
31
+ @is_parent ||= Types.parents.include?(type)
32
+ end
33
+
34
+ def default_mapping
35
+ default = {}.extend Struct::Mergeable
36
+ if is_child?
37
+ parent_child_map.each do |parent, child|
38
+ default.deep_merge! index => {'mappings' => {child => {'_parent' => {'type' => parent}}}}
39
+ end
40
+ end
41
+ default
42
+ end
43
+
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,30 @@
1
+ module Elastics
2
+ module ClassProxy
3
+ module ModelSyncer
4
+
5
+ attr_accessor :synced
6
+
7
+ def sync(*synced)
8
+ # Elastics::ActiveModel has its own way of syncing, and a Elastics::ModelSyncer cannot be synced by itself
9
+ raise ArgumentError, %(You cannot elastics.sync(self) #{context}.) \
10
+ if synced.any?{|s| s == context} && !context.include?(Elastics::ModelIndexer)
11
+ synced.each do |s|
12
+ s == context || s.is_a?(Symbol) || s.is_a?(String) || raise(ArgumentError, "self, string or symbol expected, got #{s.inspect}")
13
+ end
14
+ @synced ||= []
15
+ @synced |= synced
16
+ add_callbacks
17
+ end
18
+
19
+ def add_callbacks
20
+ context.class_eval do
21
+ raise NotImplementedError, "the class #{self} must implement :after_save and :after_destroy callbacks" \
22
+ unless respond_to?(:after_save) && respond_to?(:after_destroy)
23
+ after_save { elastics.sync }
24
+ after_destroy { elastics.sync }
25
+ end
26
+ end
27
+
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,36 @@
1
+ module Elastics
2
+ module InstanceProxy
3
+ class ActiveModel < ModelIndexer
4
+
5
+ def store(*vars)
6
+ return super unless instance.elastics_indexable? # this should never happen since elastics_indexable? returns true
7
+ meth = (id.nil? || id.empty?) ? :post_store : :put_store
8
+ Elastics.send(meth, metainfo, {:data => instance.elastics_source}, *vars)
9
+ end
10
+
11
+ def sync_self
12
+ instance.instance_eval do
13
+ if destroyed?
14
+ if @skip_destroy_callbacks
15
+ elastics.remove
16
+ else
17
+ run_callbacks :destroy do
18
+ elastics.remove
19
+ end
20
+ end
21
+ else
22
+ run_callbacks :save do
23
+ context = new_record? ? :create : :update
24
+ run_callbacks(context) do
25
+ result = context == :create ? elastics.store : elastics.store(:params => { :version => _version })
26
+ @_id = result['_id']
27
+ @_version = result['_version']
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,125 @@
1
+ module Elastics
2
+ module InstanceProxy
3
+ class ModelIndexer < ModelSyncer
4
+
5
+ # delegates :index, :is_child?, :is_parent? to class_elastics
6
+ Utils.define_delegation :to => :class_elastics,
7
+ :in => self,
8
+ :by => :module_eval,
9
+ :for => [:is_child?, :is_parent?]
10
+
11
+ # indexes the document
12
+ # usually called from after_save, you can eventually call it explicitly for example from another callback
13
+ # or whenever the DB doesn't get updated by the model
14
+ # you can also pass the :data=>elastics_source explicitly (useful for example to override the elastics_source in the model)
15
+ def store(*vars)
16
+ if instance.elastics_indexable?
17
+ Elastics.store(metainfo, {:data => instance.elastics_source}, *vars)
18
+ else
19
+ Elastics.remove(metainfo, *vars) if Elastics.get(metainfo, *vars, :raise => false)
20
+ end
21
+ end
22
+
23
+ # removes the document from the index (called from after_destroy)
24
+ def remove(*vars)
25
+ return unless instance.elastics_indexable?
26
+ Elastics.remove(metainfo, *vars)
27
+ end
28
+
29
+ # gets the document from ES
30
+ def get(*vars)
31
+ return unless instance.elastics_indexable?
32
+ Elastics.get(metainfo, *vars)
33
+ end
34
+
35
+ # like get, but it returns all the fields after a refresh
36
+ def full_get(*vars)
37
+ return unless instance.elastics_indexable?
38
+ Elastics.search_by_id(metainfo, {:refresh => true, :params => {:fields => '*,_source'}}, *vars)
39
+ end
40
+
41
+ def parent_instance
42
+ return unless is_child?
43
+ @parent_instance ||= instance.send(class_elastics.parent_association) ||
44
+ raise(MissingParentError, "missing parent instance for document #{instance.inspect}.")
45
+ end
46
+
47
+ # helper that iterates through the parent record chain
48
+ # record.elastics.each_parent{|p| p.do_something }
49
+ def each_parent
50
+ pi = parent_instance
51
+ while pi do
52
+ yield pi
53
+ pi = pi.elastics.parent_instance
54
+ end
55
+ end
56
+
57
+ def index
58
+ @index ||= instance.respond_to?(:elastics_index) ? instance.elastics_index : class_elastics.index
59
+ end
60
+ attr_writer :index
61
+
62
+ def type
63
+ @type ||= case
64
+ when instance.respond_to?(:elastics_type) then instance.elastics_type
65
+ when is_child? then class_elastics.parent_child_map[parent_instance.elastics.type]
66
+ else class_elastics.type
67
+ end
68
+ end
69
+ attr_writer :type
70
+
71
+ def id
72
+ @id ||= instance.respond_to?(:elastics_id) ? instance.elastics_id : instance.id.to_s
73
+ end
74
+
75
+ def routing
76
+ @routing ||= case
77
+ when instance.respond_to?(:elastics_routing) then instance.elastics_routing
78
+ when is_child? then parent_instance.elastics.routing
79
+ when is_parent? then create_routing
80
+ end
81
+ end
82
+ attr_writer :routing
83
+
84
+ def parent
85
+ @parent ||= case
86
+ when instance.respond_to?(:elastics_parent) then instance.elastics_parent
87
+ when is_child? then parent_instance.id.to_s
88
+ else nil
89
+ end
90
+ end
91
+ attr_writer :parent
92
+
93
+ def metainfo
94
+ meta = Vars.new( :index => index, :type => type, :id => id )
95
+ params = {}
96
+ params[:routing] = routing if routing
97
+ params[:parent] = parent if parent
98
+ meta.merge!(:params => params) unless params.empty?
99
+ meta
100
+ end
101
+
102
+ def sync_self
103
+ instance.destroyed? ? remove : store
104
+ end
105
+
106
+ private
107
+
108
+ BASE62_DIGITS = ('0'..'9').to_a + ('A'..'Z').to_a + ('a'..'z').to_a
109
+
110
+ def create_routing
111
+ string = [index, type, id].join
112
+ remainder = Digest::MD5.hexdigest(string).to_i(16)
113
+ result = []
114
+ max_power = ( Math.log(remainder) / Math.log(62) ).floor
115
+ max_power.downto(0) do |power|
116
+ digit, remainder = remainder.divmod(62**power)
117
+ result << digit
118
+ end
119
+ result << remainder if remainder > 0
120
+ result.map{|digit| BASE62_DIGITS[digit]}.join
121
+ end
122
+
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,54 @@
1
+ module Elastics
2
+ module InstanceProxy
3
+ class ModelSyncer
4
+
5
+ attr_reader :instance, :class_elastics
6
+
7
+ def initialize(instance)
8
+ @instance = instance
9
+ @class_elastics = instance.class.elastics
10
+ end
11
+
12
+ def sync(*trail)
13
+ return if trail.include?(uid) || class_elastics.synced.nil?
14
+ trail << uid
15
+ class_elastics.synced.each do |synced|
16
+ case
17
+ # sync self
18
+ when synced == instance.class
19
+ sync_self
20
+ # sync :author, :comments
21
+ # works for all association types, if the instances have a #elastics proxy
22
+ when synced.is_a?(Symbol)
23
+ to_sync = instance.send(synced)
24
+ if to_sync.respond_to?(:each)
25
+ to_sync.each { |s| s.elastics.sync(*trail) }
26
+ else
27
+ to_sync.elastics.sync(*trail)
28
+ end
29
+ # sync 'blog'
30
+ # polymorphic: use this form only if you want to sync any specific parent type but not all
31
+ when synced.is_a?(String)
32
+ next unless synced == parent_instance.elastics.type
33
+ parent_instance.elastics.sync(*trail)
34
+ else
35
+ raise ArgumentError, "self, string or symbol expected, got #{synced.inspect}"
36
+ end
37
+ end
38
+ end
39
+
40
+ def uid
41
+ @uid ||= [instance.class, instance.id].join('-')
42
+ end
43
+
44
+ def refresh_index
45
+ class_elastics.refresh_index
46
+ end
47
+
48
+ def sync_self
49
+ # nothing to sync, since a ModelSyncer cannot sync itselfs
50
+ end
51
+
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,67 @@
1
+ module Elastics
2
+ # private module
3
+ module LiveReindex
4
+
5
+ def reindex_models(opts={})
6
+
7
+ raise NotImplementedError, 'Elastics::LiveReindex.reindex_models requires the "elastics-admin" gem. Please, install it.' \
8
+ unless defined?(Elastics::Admin)
9
+
10
+ on_each_change do |action, document|
11
+ if action == 'index'
12
+ begin
13
+ { action => document.load! }
14
+ rescue Mongoid::Errors::DocumentNotFound, ActiveRecord::RecordNotFound
15
+ nil # record already deleted
16
+ end
17
+ else
18
+ { action => document }
19
+ end
20
+ end
21
+
22
+ yield self if block_given?
23
+
24
+ # we override the on_reindex eventually set
25
+ on_reindex do
26
+ opts = opts.merge(:force => false)
27
+ ModelTasks.new(opts).import_models
28
+ end
29
+
30
+ perform(opts)
31
+ end
32
+
33
+ def reindex_active_models(opts={})
34
+
35
+ raise NotImplementedError, 'Elastics::LiveReindex.reindex_models requires the "elastics-admin" gem. PLease, install it.' \
36
+ unless defined?(Elastics::Admin)
37
+
38
+ yield self if block_given?
39
+
40
+ opts[:verbose] = true unless opts.has_key?(:verbose)
41
+ opts[:models] ||= Conf.elastics_active_models
42
+
43
+ # we override the on_reindex eventually set
44
+ on_reindex do
45
+ opts[:models].each do |model|
46
+ model = eval("::#{model}") if model.is_a?(String)
47
+ raise ArgumentError, "The model #{model.name} is not a standard Elastics::ActiveModel model" \
48
+ unless model.include?(Elastics::ActiveModel)
49
+
50
+ pbar = ProgBar.new(model.count, nil, "Model #{model}: ") if opts[:verbose]
51
+
52
+ model.find_in_batches({:raw_result => true, :params => {:fields => '*,_source'}}, opts) do |result|
53
+ batch = result['hits']['hits']
54
+ result = process_and_post_batch(batch)
55
+ pbar.process_result(result, batch.size) if opts[:verbose]
56
+ end
57
+
58
+ pbar.finish if opts[:verbose]
59
+
60
+ end
61
+ end
62
+
63
+ perform(opts)
64
+ end
65
+
66
+ end
67
+ end
@@ -0,0 +1,27 @@
1
+ module Elastics
2
+ module ModelIndexer
3
+
4
+ def self.included(base)
5
+ base.class_eval do
6
+ @elastics ||= ClassProxy::Base.new(base)
7
+ @elastics.extend(ClassProxy::ModelSyncer)
8
+ @elastics.extend(ClassProxy::ModelIndexer).init
9
+ def self.elastics; @elastics end
10
+ end
11
+ end
12
+
13
+ def elastics
14
+ @elastics ||= InstanceProxy::ModelIndexer.new(self)
15
+ end
16
+
17
+ def elastics_source
18
+ attributes.reject {|k| k.to_s =~ /^_*id$/}
19
+ end
20
+
21
+ def elastics_indexable?
22
+ true
23
+ end
24
+
25
+ end
26
+
27
+ end
@@ -0,0 +1,17 @@
1
+ module Elastics
2
+ module ModelSyncer
3
+
4
+ def self.included(base)
5
+ base.class_eval do
6
+ @elastics ||= ClassProxy::Base.new(base)
7
+ @elastics.extend(ClassProxy::ModelSyncer)
8
+ def self.elastics; @elastics end
9
+ end
10
+ end
11
+
12
+ def elastics
13
+ @elastics ||= InstanceProxy::ModelSyncer.new(self)
14
+ end
15
+
16
+ end
17
+ end
@@ -0,0 +1,116 @@
1
+ require 'elastics/tasks'
2
+
3
+ module Elastics
4
+
5
+ class Tasks
6
+ # patches the Elastics::Tasks#config_hash so it evaluates also the default mapping for models
7
+ # it modifies also the index:create task
8
+ alias_method :original_config_hash, :config_hash
9
+ def config_hash
10
+ @config_hash ||= begin
11
+ default = {}.extend Struct::Mergeable
12
+ (Conf.elastics_models + Conf.elastics_active_models).each do |m|
13
+ m = eval"::#{m}" if m.is_a?(String)
14
+ default.deep_merge! m.elastics.default_mapping
15
+ end
16
+ default.deep_merge(original_config_hash)
17
+ end
18
+ end
19
+ end
20
+
21
+ class ModelTasks < Elastics::Tasks
22
+
23
+ attr_reader :options
24
+
25
+ def initialize(overrides={})
26
+ options = Elastics::Utils.env2options *default_options.keys
27
+
28
+ options[:timeout] = options[:timeout].to_i if options[:timeout]
29
+ options[:batch_size] = options[:batch_size].to_i if options[:batch_size]
30
+ options[:models] = options[:models].split(',') if options[:models]
31
+
32
+ if options[:import_options]
33
+ import_options = {}
34
+ options[:import_options].split('&').each do |pair|
35
+ k, v = pair.split('=')
36
+ import_options[k.to_sym] = v
37
+ end
38
+ options[:import_options] = import_options
39
+ end
40
+
41
+ @options = default_options.merge(options).merge(overrides)
42
+ end
43
+
44
+ def default_options
45
+ @default_options ||= { :force => false,
46
+ :timeout => 20,
47
+ :batch_size => 1000,
48
+ :import_options => { },
49
+ :models => Conf.elastics_models,
50
+ :config_file => Conf.config_file,
51
+ :verbose => true }
52
+ end
53
+
54
+ def import_models
55
+ Conf.http_client.options[:timeout] = options[:timeout]
56
+ deleted = []
57
+ models.each do |model|
58
+ raise ArgumentError, "The model #{model.name} is not a standard Elastics::ModelIndexer model" \
59
+ unless model.include?(Elastics::ModelIndexer)
60
+ index = model.elastics.index
61
+ index = LiveReindex.prefix_index(index) if LiveReindex.should_prefix_index?
62
+
63
+ # block never called during live-reindex, since it doesn't exist
64
+ if options[:force]
65
+ unless deleted.include?(index)
66
+ delete_index(index)
67
+ deleted << index
68
+ puts "#{index} index deleted" if options[:verbose]
69
+ end
70
+ end
71
+
72
+ # block never called during live-reindex, since prefix_index creates it
73
+ unless exist?(index)
74
+ create(index)
75
+ puts "#{index} index created" if options[:verbose]
76
+ end
77
+
78
+ if defined?(Mongoid::Document) && model.include?(Mongoid::Document)
79
+ def model.find_in_batches(options={})
80
+ 0.step(count, options[:batch_size]) do |offset|
81
+ yield limit(options[:batch_size]).skip(offset).to_a
82
+ end
83
+ end
84
+ end
85
+
86
+ unless model.respond_to?(:find_in_batches)
87
+ Conf.logger.error "Model #{model} does not respond to :find_in_batches. Skipped."
88
+ next
89
+ end
90
+
91
+ pbar = ProgBar.new(model.count, options[:batch_size], "Model #{model}: ") if options[:verbose]
92
+
93
+ model.find_in_batches(:batch_size => options[:batch_size]) do |batch|
94
+ result = Elastics.post_bulk_collection(batch, options[:import_options]) || next
95
+ pbar.process_result(result, batch.size) if options[:verbose]
96
+ end
97
+
98
+ pbar.finish if options[:verbose]
99
+ end
100
+ end
101
+
102
+ private
103
+
104
+ def models
105
+ @models ||= begin
106
+ models = options[:models] || Conf.elastics_models
107
+ raise ArgumentError, 'no class defined. Please use MODELS=ClassA,ClassB ' +
108
+ 'or set the Elastics::Configuration.elastics_models properly' \
109
+ if models.nil? || models.empty?
110
+ models.map{|c| c.is_a?(String) ? eval("::#{c}") : c}
111
+ end
112
+ end
113
+
114
+ end
115
+
116
+ end
@@ -0,0 +1,15 @@
1
+ module Elastics
2
+ module RefreshCallbacks
3
+
4
+ def self.included(base)
5
+ base.class_eval do
6
+ raise NotImplementedError, "the class #{self} must implement :after_create, :after_update and :after_destroy callbacks" \
7
+ unless respond_to?(:after_save) && respond_to?(:after_destroy)
8
+ refresh = proc{ Elastics.refresh_index :index => self.class.elastics.index }
9
+ after_save &refresh
10
+ after_destroy &refresh
11
+ end
12
+ end
13
+
14
+ end
15
+ end
@@ -0,0 +1,61 @@
1
+ module Elastics
2
+ class Result
3
+ module ActiveModel
4
+
5
+ include Elastics::Result::Scope
6
+
7
+ # extend if the context include a Elastics::ActiveModel
8
+ def self.should_extend?(result)
9
+ result.variables[:context] && result.variables[:context].include?(Elastics::ActiveModel)
10
+ end
11
+
12
+ def get_docs
13
+ # super is from elastics-scopes
14
+ docs = super
15
+ return docs if variables[:raw_result]
16
+ raw_result = self
17
+ if docs.is_a?(Array)
18
+ res = docs.map {|doc| build_object(doc)}
19
+ res.extend(Struct::Paginable).setup(raw_result['hits']['total'], variables)
20
+ class << res; self end.class_eval do
21
+ define_method(:raw_result){ raw_result }
22
+ define_method(:method_missing) do |meth, *args, &block|
23
+ raw_result.respond_to?(meth) ? raw_result.send(meth, *args, &block) : super(meth, *args, &block)
24
+ end
25
+ end
26
+ res
27
+ else
28
+ build_object docs
29
+ end
30
+ end
31
+
32
+ private
33
+
34
+ def build_object(doc)
35
+ attrs = (doc['_source']||{}).merge(doc['fields']||{})
36
+ object = variables[:context].new attrs
37
+ raw_result = self
38
+ object.instance_eval do
39
+ class << self; self end.class_eval do
40
+ define_method(:raw_result){ raw_result }
41
+ define_method(:raw_document){ doc }
42
+ define_method(:respond_to?) do |*args|
43
+ doc.respond_to?(*args) || super(*args)
44
+ end
45
+ define_method(:method_missing) do |meth, *args, &block|
46
+ doc.respond_to?(meth) ? doc.send(meth, *args, &block) : super(meth, *args, &block)
47
+ end
48
+ end
49
+ @_id = doc['_id']
50
+ @_version = doc['_version']
51
+ @highlight = doc['highlight']
52
+ # load the elastics proxy before freezing
53
+ elastics
54
+ self.freeze if raw_result.variables[:params][:fields] || doc['fields']
55
+ end
56
+ object
57
+ end
58
+
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,56 @@
1
+ module Elastics
2
+ class Result
3
+
4
+ # adds sugar to documents with the following structure:
5
+ #
6
+ # {
7
+ # "_index" : "twitter",
8
+ # "_type" : "tweet",
9
+ # "_id" : "1",
10
+ # }
11
+
12
+ module DocumentLoader
13
+
14
+ module ModelClasses
15
+ extend self
16
+ # maps all the index/types to the ruby class
17
+ def map
18
+ @map ||= begin
19
+ map = {}
20
+ (Conf.elastics_models + Conf.elastics_active_models).each do |m|
21
+ m = eval("::#{m}") if m.is_a?(String)
22
+ indices = m.elastics.index.is_a?(Array) ? m.elastics.index : [m.elastics.index]
23
+ types = m.elastics.type.is_a?(Array) ? m.elastics.type : [m.elastics.type]
24
+ indices.each do |i|
25
+ types.each { |t| map["#{i}/#{t}"] = m }
26
+ end
27
+ end
28
+ map
29
+ end
30
+ end
31
+ end
32
+
33
+ # extend if result has a structure like a document
34
+ def self.should_extend?(result)
35
+ result.is_a? Document
36
+ end
37
+
38
+ def model_class
39
+ @model_class ||= ModelClasses.map["#{index_basename}/#{self['_type']}"]
40
+ end
41
+
42
+ def load
43
+ model_class.find(self['_id']) if model_class
44
+ end
45
+
46
+ def load!
47
+ raise DocumentMappingError, "the '#{index_basename}/#{self['_type']}' document cannot be mapped to any class." \
48
+ unless model_class
49
+ model_class.find self['_id']
50
+ end
51
+
52
+ end
53
+
54
+ end
55
+ end
56
+
@@ -0,0 +1,40 @@
1
+ module Elastics
2
+ class Result
3
+ module SearchLoader
4
+
5
+ # extend if result is a Search or MultiGet
6
+ def self.should_extend?(result)
7
+ result.is_a?(Search) || result.is_a?(MultiGet)
8
+ end
9
+
10
+ # extend the collection on extend
11
+ def self.extended(result)
12
+ result.collection.each { |h| h.extend(DocumentLoader) }
13
+ end
14
+
15
+ def loaded_collection
16
+ @loaded_collection ||= begin
17
+ records = []
18
+ # returns a structure like {Comment=>[{"_id"=>"123", ...}, {...}], BlogPost=>[...]}
19
+ h = Utils.group_array_by(collection) do |d|
20
+ d.model_class
21
+ end
22
+ h.each do |klass, docs|
23
+ records |= klass.find(docs.map(&:_id))
24
+ end
25
+ class_ids = collection.map { |d| [d.model_class.to_s, d._id] }
26
+ # Reorder records to preserve order from search results
27
+ records = class_ids.map do |class_str, id|
28
+ records.detect do |record|
29
+ record.class.to_s == class_str && record.id.to_s == id.to_s
30
+ end
31
+ end
32
+ records.extend Struct::Paginable
33
+ records.setup(collection.total_entries, variables)
34
+ records
35
+ end
36
+ end
37
+
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,38 @@
1
+ module Elastics
2
+ module Struct
3
+ # allows deep merge between Hashes
4
+ module Mergeable
5
+
6
+ def deep_merge(*hashes)
7
+ merged = deep_dup
8
+ hashes.each {|h2| merged.replace(deep_merge_hash(merged,h2))}
9
+ merged
10
+ end
11
+
12
+ def deep_merge!(*hashes)
13
+ replace deep_merge(*hashes)
14
+ end
15
+
16
+ def deep_dup
17
+ Marshal.load(Marshal.dump(self))
18
+ end
19
+
20
+ private
21
+
22
+ def deep_merge_hash(h1, h2)
23
+ h2 ||= {}
24
+ h1.merge(h2) do |key, oldval, newval|
25
+ case
26
+ when oldval.is_a?(::Hash) && newval.is_a?(::Hash)
27
+ deep_merge_hash(oldval, newval)
28
+ when oldval.is_a?(::Array) && newval.is_a?(::Array)
29
+ oldval + newval
30
+ else
31
+ newval
32
+ end
33
+ end
34
+ end
35
+
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,40 @@
1
+ require 'elastics'
2
+ require 'elastics-scopes'
3
+ require 'active_attr'
4
+
5
+ require 'elastics/struct/mergeable'
6
+
7
+ require 'elastics/class_proxy/model_syncer'
8
+ require 'elastics/instance_proxy/model_syncer'
9
+ require 'elastics/model_syncer'
10
+
11
+ require 'elastics/class_proxy/model_indexer'
12
+ require 'elastics/instance_proxy/model_indexer'
13
+ require 'elastics/model_indexer'
14
+
15
+ require 'elastics/active_model/timestamps'
16
+ require 'elastics/active_model/attachment'
17
+ require 'elastics/active_model/inspection'
18
+ require 'elastics/active_model/storage'
19
+ require 'elastics/class_proxy/active_model'
20
+ require 'elastics/instance_proxy/active_model'
21
+ require 'elastics/active_model'
22
+
23
+ require 'elastics/refresh_callbacks'
24
+
25
+ require 'elastics/live_reindex_model'
26
+
27
+ require 'elastics/result/document_loader'
28
+ require 'elastics/result/search_loader'
29
+ require 'elastics/result/active_model'
30
+
31
+ require 'elastics/model_tasks'
32
+
33
+ Elastics::LIB_PATHS << File.dirname(__FILE__)
34
+
35
+ # get_docs calls super so we make sure the result is extended by Scope first
36
+ Elastics::Conf.result_extenders |= [ Elastics::Result::DocumentLoader,
37
+ Elastics::Result::SearchLoader,
38
+ Elastics::Result::ActiveModel ]
39
+ Elastics::Conf.elastics_models = []
40
+ Elastics::Conf.elastics_active_models = []
data/lib/tasks.rake ADDED
@@ -0,0 +1,12 @@
1
+ require 'elastics-models'
2
+
3
+ env = defined?(Rails) ? :environment : []
4
+
5
+ namespace :'elastics-client' do
6
+
7
+ desc 'imports from an ActiveRecord or Mongoid models'
8
+ task(:import => env) { Elastics::ModelTasks.new.import_models }
9
+
10
+ task(:reset_redis_keys) { Elastics::Redis.reset_keys }
11
+
12
+ end
metadata ADDED
@@ -0,0 +1,122 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: elastics-models
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.4
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Domizio Demichelis
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-08-19 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: elastics-client
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - '='
20
+ - !ruby/object:Gem::Version
21
+ version: 1.0.4
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - '='
28
+ - !ruby/object:Gem::Version
29
+ version: 1.0.4
30
+ - !ruby/object:Gem::Dependency
31
+ name: elastics-scopes
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - '='
36
+ - !ruby/object:Gem::Version
37
+ version: 1.0.4
38
+ type: :runtime
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - '='
44
+ - !ruby/object:Gem::Version
45
+ version: 1.0.4
46
+ - !ruby/object:Gem::Dependency
47
+ name: active_attr
48
+ requirement: !ruby/object:Gem::Requirement
49
+ none: false
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: 0.6.0
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: 0.6.0
62
+ description: Provides ActiveRecord, Mongoid, ActiveModel and elasticsearch-mapper-attachment
63
+ integrations, cross syncing, parent/child relationships, bulk-import, live-reindex
64
+ of models, ...
65
+ email: dd.nexus@gmail.com
66
+ executables: []
67
+ extensions: []
68
+ extra_rdoc_files: []
69
+ files:
70
+ - LICENSE
71
+ - README.md
72
+ - VERSION
73
+ - elastics-models.gemspec
74
+ - lib/elastics-models.rb
75
+ - lib/elastics/active_model.rb
76
+ - lib/elastics/active_model/attachment.rb
77
+ - lib/elastics/active_model/inspection.rb
78
+ - lib/elastics/active_model/storage.rb
79
+ - lib/elastics/active_model/timestamps.rb
80
+ - lib/elastics/class_proxy/active_model.rb
81
+ - lib/elastics/class_proxy/model_indexer.rb
82
+ - lib/elastics/class_proxy/model_syncer.rb
83
+ - lib/elastics/instance_proxy/active_model.rb
84
+ - lib/elastics/instance_proxy/model_indexer.rb
85
+ - lib/elastics/instance_proxy/model_syncer.rb
86
+ - lib/elastics/live_reindex_model.rb
87
+ - lib/elastics/model_indexer.rb
88
+ - lib/elastics/model_syncer.rb
89
+ - lib/elastics/model_tasks.rb
90
+ - lib/elastics/refresh_callbacks.rb
91
+ - lib/elastics/result/active_model.rb
92
+ - lib/elastics/result/document_loader.rb
93
+ - lib/elastics/result/search_loader.rb
94
+ - lib/elastics/struct/mergeable.rb
95
+ - lib/tasks.rake
96
+ homepage: http://elastics.github.io/elastics
97
+ licenses:
98
+ - MIT
99
+ post_install_message:
100
+ rdoc_options:
101
+ - --charset=UTF-8
102
+ require_paths:
103
+ - lib
104
+ required_ruby_version: !ruby/object:Gem::Requirement
105
+ none: false
106
+ requirements:
107
+ - - ! '>='
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
110
+ required_rubygems_version: !ruby/object:Gem::Requirement
111
+ none: false
112
+ requirements:
113
+ - - ! '>='
114
+ - !ruby/object:Gem::Version
115
+ version: 1.3.6
116
+ requirements: []
117
+ rubyforge_project:
118
+ rubygems_version: 1.8.25
119
+ signing_key:
120
+ specification_version: 3
121
+ summary: Transparently integrates your models with one or more elasticsearch indices.
122
+ test_files: []