flex-models 1.0.1

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012-2013 by Domizio Demichelis
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,24 @@
+# Flex-model
+
+Transparently integrates your models with one or more elasticsearch indices
+
+## Links
+
+* [Flex Repository](https://github.com/ddnexus/flex)
+* [Flex Project (Global Documentation)](http://ddnexus.github.io/flex/doc/)
+* [flex-models Gem (Specific Documentation)](http://ddnexus.github.io/flex/doc/4-flex-models)
+* [Issues](https://github.com/ddnexus/flex-models/issues)
+* [Pull Requests](https://github.com/ddnexus/flex-models/pulls)
+
+## Branches
+
+The master branch reflects the last published gem. Then you may find a next-version branch (named after the version string), with the commits that will be merged in master just before publishing the next gem version. The next-version branch may get rebased or force pushed.
+
+## Credits
+
+Special thanks for their sponsorship to [Escalate Media](http://www.escalatemedia.com) and [Barquin International](http://www.barquin.com).
+
+## Copyright
+
+Copyright (c) 2012-2013 by [Domizio Demichelis](mailto://dd.nexus@gmail.com)<br>
+See [LICENSE](https://github.com/ddnexus/flex-models/blob/master/LICENSE) for details.

data/VERSION

@@ -0,0 +1 @@
+1.0.1

data/flex-models.gemspec

@@ -0,0 +1,22 @@
+require 'date'
+version = File.read(File.expand_path('../VERSION', __FILE__)).strip
+
+Gem::Specification.new do |s|
+  s.name                      = 'flex-models'
+  s.summary                   = 'Transparently integrates your models with one or more elasticsearch indices.'
+  s.description               = 'Provides ActiveRecord, Mongoid, ActiveModel and elasticsearch-mapper-attachment integrations, cross syncing, parent/child relationships, bulk-import, live-reindex of models, ...'
+  s.homepage                  = 'http://github.com/ddnexus/flex-models'
+  s.authors                   = ["Domizio Demichelis"]
+  s.email                     = 'dd.nexus@gmail.com'
+  s.extra_rdoc_files          = %w[README.md]
+  s.files                     = `git ls-files -z`.split("\0")
+  s.version                   = version
+  s.date                      = Date.today.to_s
+  s.required_rubygems_version = ">= 1.3.6"
+  s.rdoc_options              = %w[--charset=UTF-8]
+
+  s.add_runtime_dependency 'flex',        version
+  s.add_runtime_dependency 'flex-scopes', version
+
+  s.add_runtime_dependency 'active_attr', '>= 0.6.0'
+end

data/lib/flex/active_model/attachment.rb

@@ -0,0 +1,42 @@
+require 'base64'
+module Flex
+  module ActiveModel
+    module Attachment
+
+      # defines accessors for <attachment_field_name>
+      # if you omit the arguments it uses :attachment as the <attachment_field_name>
+      # you can also pass other properties that will be merged with the default property for attachment
+      # this will automatically add a :<attachment_field_name>_scope scope which will add
+      # all the meta fields (title, author, ...) to the returned fields, exluding the <attachment_field_name> field itself
+      # and including all the other attributes declared before it. For that reason you may want to declare it as
+      # the latest attribute.
+
+      def attribute_attachment(*args)
+        name  = args.first.is_a?(Symbol) ? args.shift : :attachment
+        props = {:properties => { 'type'   => 'attachment',
+                                  'fields' => { name.to_s      => { 'store' => 'yes', 'term_vector' => 'with_positions_offsets' },
+                                                'title'        => { 'store' => 'yes' },
+                                                'author'       => { 'store' => 'yes' },
+                                                'name'         => { 'store' => 'yes' },
+                                                'content_type' => { 'store' => 'yes' },
+                                                'date'         => { 'store' => 'yes' },
+                                                'keywords'     => { 'store' => 'yes' }
+                                              }
+                                }
+                }
+        props.extend(Struct::Mergeable).deep_merge! args.first if args.first.is_a?(Hash)
+
+        scope :"#{name}_scope", fields("#{name}.title",
+                                       "#{name}.author",
+                                       "#{name}.name",
+                                       "#{name}.content_type",
+                                       "#{name}.date",
+                                       "#{name}.keywords",
+                                       *attributes.keys)
+        attribute name, props
+
+      end
+
+    end
+  end
+end

data/lib/flex/active_model/inspection.rb

@@ -0,0 +1,21 @@
+module Flex
+  module ActiveModel
+    module Inspection
+
+      def inspect
+        descriptions   = [%(_id: #{@_id.inspect}), %(_version: #{@_version})]
+        all_attributes = if respond_to?(:raw_document)
+                           reader_keys = raw_document.send(:readers).keys.map(&:to_s)
+                           # we send() the readers, so they will reflect an eventual overriding
+                           Hash[ reader_keys.map{ |k| [k, send(k)] } ].merge(attributes)
+                         else
+                           attributes
+                         end
+        descriptions << all_attributes.sort.map { |key, value| "#{key}: #{value.inspect}" }
+        separator = " " unless descriptions.empty?
+        "#<#{self.class.name}#{separator}#{descriptions.join(", ")}>"
+      end
+
+    end
+  end
+end

data/lib/flex/active_model/storage.rb

@@ -0,0 +1,127 @@
+module Flex
+  module ActiveModel
+
+    class DocumentInvalidError < StandardError; end
+
+    module Storage
+
+      module ClassMethods
+
+        def create(args={})
+          document = new(args)
+          return false unless document.valid?
+          document.save
+        end
+
+      end
+
+
+      module InstanceMethods
+
+        def reload
+          document        = flex.get
+          self.attributes = document['_source']
+          @_id            = document['_id']
+          @_version       = document['_version']
+        end
+
+        def save(options={})
+          perform_validations(options) ? do_save : false
+        end
+
+        def save!(options={})
+          perform_validations(options) ? do_save : raise(DocumentInvalidError, errors.full_messages.join(", "))
+        end
+
+        # Optimistic Lock Update
+        #
+        #    doc.safe_update do |d|
+        #      d.amount += 100
+        #    end
+        #
+        # if you are trying to update a stale object, the block is yielded again with a fresh reloaded document and the
+        # document is saved only when it is not stale anymore (i.e. the _version has not changed since it has been loaded)
+        # read: http://www.elasticsearch.org/blog/2011/02/08/versioning.html
+        #
+        def safe_update(options={}, &block)
+          perform_validations(options) ? lock_update(&block) : false
+        end
+
+        def safe_update!(options={}, &block)
+          perform_validations(options) ? lock_update(&block) : raise(DocumentInvalidError, errors.full_messages.join(", "))
+        end
+
+        def valid?(context = nil)
+          context ||= (new_record? ? :create : :update)
+          output = super(context)
+          errors.empty? && output
+        end
+
+        def destroy
+          @destroyed = true
+          flex.sync
+          self.freeze
+        end
+
+        def delete
+          @skip_destroy_callbacks = true
+          destroy
+        end
+
+        def merge_attributes(attributes)
+          attributes.each {|name, value| send "#{name}=", value }
+        end
+
+        def update_attributes(attributes)
+          merge_attributes(attributes)
+          save
+        end
+
+        def destroyed?
+          !!@destroyed
+        end
+
+        def persisted?
+          !(new_record? || destroyed?)
+        end
+
+        def new_record?
+          !@_id || !@_version
+        end
+
+      private
+
+        def do_save
+          flex.sync
+          self
+        end
+
+        def lock_update
+          begin
+            yield self
+            flex.sync
+          rescue Flex::HttpError => e
+            if e.status == 409
+              reload
+              retry
+            else
+              raise
+            end
+          end
+          self
+        end
+
+      protected
+
+        def perform_validations(options={})
+          perform_validation = options[:validate] != false
+          perform_validation ? valid?(options[:context]) : true
+        end
+
+      end
+
+    end
+
+
+  end
+end

data/lib/flex/active_model/timestamps.rb

@@ -0,0 +1,22 @@
+module Flex
+  module ActiveModel
+    module Timestamps
+
+      def attribute_timestamps(props={})
+        attribute_created_at props
+        attribute_updated_at props
+      end
+
+      def attribute_created_at(props={})
+        attribute :created_at, {:type => DateTime}.merge(props)
+        before_create { self.created_at = Time.now.utc }
+      end
+
+      def attribute_updated_at(props={})
+        attribute :updated_at, {:type => DateTime}.merge(props)
+        before_save { self.updated_at = Time.now.utc }
+      end
+
+    end
+  end
+end

data/lib/flex/active_model.rb

@@ -0,0 +1,47 @@
+module Flex
+  module ActiveModel
+
+    attr_reader :_version, :_id, :highlight
+    alias_method :id, :_id
+
+    def self.included(base)
+      base.class_eval do
+        @flex ||= ClassProxy::Base.new(base)
+        @flex.extend(ClassProxy::ModelSyncer)
+        @flex.extend(ClassProxy::ModelIndexer).init
+        @flex.extend(ClassProxy::ActiveModel).init :params => {:version => true}
+        def self.flex; @flex end
+        flex.synced = [self]
+
+        include Scopes
+        include ActiveAttr::Model
+
+        extend  ::ActiveModel::Callbacks
+        define_model_callbacks :create, :update, :save, :destroy
+
+        include Storage::InstanceMethods
+        extend  Storage::ClassMethods
+        include Inspection
+        extend  Timestamps
+        extend  Attachment
+      end
+    end
+
+    def flex
+      @flex ||= InstanceProxy::ActiveModel.new(self)
+    end
+
+    def flex_source
+      attributes
+    end
+
+    def flex_indexable?
+      true
+    end
+
+    def method_missing(meth, *args, &block)
+      raw_document.respond_to?(meth) ? raw_document.send(meth) : super
+    end
+
+  end
+end

data/lib/flex/class_proxy/active_model.rb

@@ -0,0 +1,35 @@
+module Flex
+  module ClassProxy
+    module ActiveModel
+
+      def init(*vars)
+        variables.deep_merge! *vars
+      end
+
+      def default_mapping
+        props = { }
+        context.attributes.each do |name, attr|
+          options     = attr.send(:options)
+          props[name] = case
+                        when options.has_key?(:properties)
+                          Utils.keyfy(:to_s, attr.send(:options)[:properties])
+                        when options.has_key?(:not_analyzed) && options[:not_analyzed] ||
+                             options.has_key?(:analyzed)     && !options[:analyzed]
+                          { 'type' => 'string', 'index' => 'not_analyzed' }
+                        when options[:type] == DateTime
+                          { 'type' => 'date', 'format' => 'dateOptionalTime' }
+                        else
+                          next
+                        end
+        end
+        props.empty? ? super :  super.deep_merge(index => {'mappings' => {type => {'properties' => props}}})
+      end
+
+      # overrides the ModelSyncer#add_callbacks
+      def add_callbacks
+        # no callbacks to add, since it calls flex.sync on save and destroy
+      end
+
+    end
+  end
+end

data/lib/flex/class_proxy/model_indexer.rb

@@ -0,0 +1,46 @@
+module Flex
+  module ClassProxy
+    module ModelIndexer
+
+      module Types
+        extend self
+
+        attr_accessor :parents
+        @parents = []
+      end
+
+      attr_reader :parent_association, :parent_child_map
+
+      def init
+        variables.deep_merge! :type  => Utils.class_name_to_type(context.name)
+      end
+
+      def parent(parent_association, map)
+        @parent_association = parent_association
+        Types.parents      |= map.keys.map(&:to_s)
+        self.type           = map.values.map(&:to_s)
+        @parent_child_map   = map
+        @is_child           = true
+      end
+
+      def is_child?
+        !!@is_child
+      end
+
+      def is_parent?
+        @is_parent ||= Types.parents.include?(type)
+      end
+
+      def default_mapping
+        default = {}.extend Struct::Mergeable
+        if is_child?
+          parent_child_map.each do |parent, child|
+            default.deep_merge! index => {'mappings' => {child => {'_parent' => {'type' => parent}}}}
+          end
+        end
+        default
+      end
+
+    end
+  end
+end

data/lib/flex/class_proxy/model_syncer.rb

@@ -0,0 +1,30 @@
+module Flex
+  module ClassProxy
+    module ModelSyncer
+
+      attr_accessor :synced
+
+      def sync(*synced)
+        # Flex::ActiveModel has its own way of syncing, and a Flex::ModelSyncer cannot be synced by itself
+        raise ArgumentError, %(You cannot flex.sync(self) #{context}.) \
+              if synced.any?{|s| s == context} && !context.include?(Flex::ModelIndexer)
+        synced.each do |s|
+          s == context || s.is_a?(Symbol) || s.is_a?(String) || raise(ArgumentError, "self, string or symbol expected, got #{s.inspect}")
+        end
+        @synced ||= []
+        @synced  |= synced
+        add_callbacks
+      end
+
+      def add_callbacks
+        context.class_eval do
+          raise NotImplementedError, "the class #{self} must implement :after_save and :after_destroy callbacks" \
+                unless respond_to?(:after_save) && respond_to?(:after_destroy)
+          after_save    { flex.sync }
+          after_destroy { flex.sync }
+        end
+      end
+
+    end
+  end
+end

data/lib/flex/instance_proxy/active_model.rb

@@ -0,0 +1,36 @@
+module Flex
+  module InstanceProxy
+    class ActiveModel < ModelIndexer
+
+      def store(*vars)
+        return super unless instance.flex_indexable? # this should never happen since flex_indexable? returns true
+        meth = (id.nil? || id.empty?) ? :post_store : :put_store
+        Flex.send(meth, metainfo, {:data => instance.flex_source}, *vars)
+      end
+
+      def sync_self
+        instance.instance_eval do
+          if destroyed?
+            if @skip_destroy_callbacks
+              flex.remove
+            else
+              run_callbacks :destroy do
+                flex.remove
+              end
+            end
+          else
+            run_callbacks :save do
+              context = new_record? ? :create : :update
+              run_callbacks(context) do
+                result    = context == :create ? flex.store : flex.store(:params => { :version => _version })
+                @_id      = result['_id']
+                @_version = result['_version']
+              end
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/flex/instance_proxy/model_indexer.rb

@@ -0,0 +1,125 @@
+module Flex
+  module InstanceProxy
+    class ModelIndexer < ModelSyncer
+
+      # delegates :index, :is_child?, :is_parent? to class_flex
+      Utils.define_delegation :to  => :class_flex,
+                              :in  => self,
+                              :by  => :module_eval,
+                              :for => [:is_child?, :is_parent?]
+
+      # indexes the document
+      # usually called from after_save, you can eventually call it explicitly for example from another callback
+      # or whenever the DB doesn't get updated by the model
+      # you can also pass the :data=>flex_source explicitly (useful for example to override the flex_source in the model)
+      def store(*vars)
+        if instance.flex_indexable?
+          Flex.store(metainfo, {:data => instance.flex_source}, *vars)
+        else
+          Flex.remove(metainfo, *vars) if Flex.get(metainfo, *vars, :raise => false)
+        end
+      end
+
+      # removes the document from the index (called from after_destroy)
+      def remove(*vars)
+        return unless instance.flex_indexable?
+        Flex.remove(metainfo, *vars)
+      end
+
+      # gets the document from ES
+      def get(*vars)
+        return unless instance.flex_indexable?
+        Flex.get(metainfo, *vars)
+      end
+
+      # like get, but it returns all the fields after a refresh
+      def full_get(*vars)
+        return unless instance.flex_indexable?
+        Flex.search_by_id(metainfo, {:refresh => true, :params => {:fields => '*,_source'}}, *vars)
+      end
+
+      def parent_instance
+        return unless is_child?
+        @parent_instance ||= instance.send(class_flex.parent_association) ||
+                               raise(MissingParentError, "missing parent instance for document #{instance.inspect}.")
+      end
+
+      # helper that iterates through the parent record chain
+      # record.flex.each_parent{|p| p.do_something }
+      def each_parent
+        pi = parent_instance
+        while pi do
+          yield pi
+          pi = pi.flex.parent_instance
+        end
+      end
+
+      def index
+        @index ||= instance.respond_to?(:flex_index) ? instance.flex_index : class_flex.index
+      end
+      attr_writer :index
+
+      def type
+        @type ||= case
+                  when instance.respond_to?(:flex_type) then instance.flex_type
+                  when is_child?                        then class_flex.parent_child_map[parent_instance.flex.type]
+                  else                                       class_flex.type
+                  end
+      end
+      attr_writer :type
+
+      def id
+        @id ||= instance.respond_to?(:flex_id) ? instance.flex_id : instance.id.to_s
+      end
+
+      def routing
+        @routing ||= case
+                     when instance.respond_to?(:flex_routing) then instance.flex_routing
+                     when is_child?                           then parent_instance.flex.routing
+                     when is_parent?                          then create_routing
+                     end
+      end
+      attr_writer :routing
+
+      def parent
+        @parent ||= case
+                    when instance.respond_to?(:flex_parent) then instance.flex_parent
+                    when is_child?                          then parent_instance.id.to_s
+                    else nil
+                    end
+      end
+      attr_writer :parent
+
+      def metainfo
+        meta = Vars.new( :index => index, :type => type, :id => id )
+        params = {}
+        params[:routing] = routing if routing
+        params[:parent]  = parent  if parent
+        meta.merge!(:params => params) unless params.empty?
+        meta
+      end
+
+      def sync_self
+        instance.destroyed? ? remove : store
+      end
+
+      private
+
+      BASE62_DIGITS = ('0'..'9').to_a + ('A'..'Z').to_a + ('a'..'z').to_a
+
+      def create_routing
+        string    = [index, type, id].join
+        remainder = Digest::MD5.hexdigest(string).to_i(16)
+        result    = []
+        max_power = ( Math.log(remainder) / Math.log(62) ).floor
+        max_power.downto(0) do |power|
+          digit, remainder = remainder.divmod(62**power)
+          result << digit
+        end
+        result << remainder if remainder > 0
+        result.map{|digit| BASE62_DIGITS[digit]}.join
+      end
+
+    end
+  end
+end

data/lib/flex/instance_proxy/model_syncer.rb

@@ -0,0 +1,54 @@
+module Flex
+  module InstanceProxy
+    class ModelSyncer
+
+      attr_reader :instance, :class_flex
+
+      def initialize(instance)
+        @instance   = instance
+        @class_flex = instance.class.flex
+      end
+
+      def sync(*trail)
+        return if trail.include?(uid) || class_flex.synced.nil?
+        trail << uid
+        class_flex.synced.each do |synced|
+          case
+          # sync self
+          when synced == instance.class
+            sync_self
+          # sync :author, :comments
+          # works for all association types, if the instances have a #flex proxy
+          when synced.is_a?(Symbol)
+            to_sync = instance.send(synced)
+            if to_sync.respond_to?(:each)
+              to_sync.each { |s| s.flex.sync(*trail) }
+            else
+              to_sync.flex.sync(*trail)
+            end
+          # sync 'blog'
+          # polymorphic: use this form only if you want to sync any specific parent type but not all
+          when synced.is_a?(String)
+            next unless synced == parent_instance.flex.type
+            parent_instance.flex.sync(*trail)
+          else
+            raise ArgumentError, "self, string or symbol expected, got #{synced.inspect}"
+          end
+        end
+      end
+
+      def uid
+        @uid ||= [instance.class, instance.id].join('-')
+      end
+
+      def refresh_index
+        class_flex.refresh_index
+      end
+
+      def sync_self
+        # nothing to sync, since a ModelSyncer cannot sync itselfs
+      end
+
+    end
+  end
+end

data/lib/flex/live_reindex_model.rb

@@ -0,0 +1,67 @@
+module Flex
+  # private module
+  module LiveReindex
+
+    def reindex_models(opts={})
+
+      raise NotImplementedError, 'Flex::LiveReindex.reindex_models requires the "flex-admin" gem. Please, install it.' \
+            unless defined?(Flex::Admin)
+
+      on_each_change do |action, document|
+        if action == 'index'
+          begin
+            { action => document.load! }
+          rescue Mongoid::Errors::DocumentNotFound, ActiveRecord::RecordNotFound
+            nil # record already deleted
+          end
+        else
+          { action => document }
+        end
+      end
+
+      yield self if block_given?
+
+      # we override the on_reindex eventually set
+      on_reindex do
+        opts = opts.merge(:force => false)
+        ModelTasks.new(opts).import_models
+      end
+
+      perform(opts)
+    end
+
+    def reindex_active_models(opts={})
+
+      raise NotImplementedError, 'Flex::LiveReindex.reindex_models requires the "flex-admin" gem. PLease, install it.' \
+            unless defined?(Flex::Admin)
+
+      yield self if block_given?
+
+      opts[:verbose]  = true unless opts.has_key?(:verbose)
+      opts[:models] ||= Conf.flex_active_models
+
+      # we override the on_reindex eventually set
+      on_reindex do
+        opts[:models].each do |model|
+          model = eval("::#{model}") if model.is_a?(String)
+          raise ArgumentError, "The model #{model.name} is not a standard Flex::ActiveModel model" \
+                unless model.include?(Flex::ActiveModel)
+
+          pbar = ProgBar.new(model.count, nil, "Model #{model}: ") if opts[:verbose]
+
+          model.find_in_batches({:raw_result => true, :params => {:fields => '*,_source'}}, opts) do |result|
+            batch  = result['hits']['hits']
+            result = process_and_post_batch(batch)
+            pbar.process_result(result, batch.size) if opts[:verbose]
+          end
+
+          pbar.finish if opts[:verbose]
+
+        end
+      end
+
+      perform(opts)
+    end
+
+  end
+end

data/lib/flex/model_indexer.rb

@@ -0,0 +1,27 @@
+module Flex
+  module ModelIndexer
+
+    def self.included(base)
+      base.class_eval do
+        @flex ||= ClassProxy::Base.new(base)
+        @flex.extend(ClassProxy::ModelSyncer)
+        @flex.extend(ClassProxy::ModelIndexer).init
+        def self.flex; @flex end
+      end
+    end
+
+    def flex
+      @flex ||= InstanceProxy::ModelIndexer.new(self)
+    end
+
+    def flex_source
+      attributes.reject {|k| k.to_s =~ /^_*id$/}
+    end
+
+    def flex_indexable?
+      true
+    end
+
+  end
+
+end

data/lib/flex/model_syncer.rb

@@ -0,0 +1,17 @@
+module Flex
+  module ModelSyncer
+
+    def self.included(base)
+      base.class_eval do
+        @flex ||= ClassProxy::Base.new(base)
+        @flex.extend(ClassProxy::ModelSyncer)
+        def self.flex; @flex end
+      end
+    end
+
+    def flex
+      @flex ||= InstanceProxy::ModelSyncer.new(self)
+    end
+
+  end
+end

data/lib/flex/model_tasks.rb

@@ -0,0 +1,116 @@
+require 'flex/tasks'
+
+module Flex
+
+  class Tasks
+    # patches the Flex::Tasks#config_hash so it evaluates also the default mapping for models
+    # it modifies also the index:create task
+    alias_method :original_config_hash, :config_hash
+    def config_hash
+      @config_hash ||= begin
+                         default = {}.extend Struct::Mergeable
+                         (Conf.flex_models + Conf.flex_active_models).each do |m|
+                           m = eval"::#{m}" if m.is_a?(String)
+                           default.deep_merge! m.flex.default_mapping
+                         end
+                         default.deep_merge(original_config_hash)
+                       end
+    end
+  end
+
+  class ModelTasks < Flex::Tasks
+
+    attr_reader :options
+
+    def initialize(overrides={})
+      options = Flex::Utils.env2options *default_options.keys
+
+      options[:timeout]    = options[:timeout].to_i      if options[:timeout]
+      options[:batch_size] = options[:batch_size].to_i   if options[:batch_size]
+      options[:models]     = options[:models].split(',') if options[:models]
+
+      if options[:import_options]
+        import_options = {}
+        options[:import_options].split('&').each do |pair|
+          k, v  = pair.split('=')
+          import_options[k.to_sym] = v
+        end
+        options[:import_options] = import_options
+      end
+
+      @options = default_options.merge(options).merge(overrides)
+    end
+
+    def default_options
+      @default_options ||= { :force          => false,
+                             :timeout        => 20,
+                             :batch_size     => 1000,
+                             :import_options => { },
+                             :models         => Conf.flex_models,
+                             :config_file    => Conf.config_file,
+                             :verbose        => true }
+    end
+
+    def import_models
+      Conf.http_client.options[:timeout] = options[:timeout]
+      deleted = []
+      models.each do |model|
+        raise ArgumentError, "The model #{model.name} is not a standard Flex::ModelIndexer model" \
+              unless model.include?(Flex::ModelIndexer)
+        index = model.flex.index
+        index = LiveReindex.prefix_index(index) if LiveReindex.should_prefix_index?
+
+        # block never called during live-reindex, since it doesn't exist
+        if options[:force]
+          unless deleted.include?(index)
+            delete_index(index)
+            deleted << index
+            puts "#{index} index deleted" if options[:verbose]
+          end
+        end
+
+        # block never called during live-reindex, since prefix_index creates it
+        unless exist?(index)
+          create(index)
+          puts "#{index} index created" if options[:verbose]
+        end
+
+        if defined?(Mongoid::Document) && model.include?(Mongoid::Document)
+          def model.find_in_batches(options={})
+            0.step(count, options[:batch_size]) do |offset|
+              yield limit(options[:batch_size]).skip(offset).to_a
+            end
+          end
+        end
+
+        unless model.respond_to?(:find_in_batches)
+          Conf.logger.error "Model #{model} does not respond to :find_in_batches. Skipped."
+          next
+        end
+
+        pbar = ProgBar.new(model.count, options[:batch_size], "Model #{model}: ") if options[:verbose]
+
+        model.find_in_batches(:batch_size => options[:batch_size]) do |batch|
+          result = Flex.post_bulk_collection(batch, options[:import_options]) || next
+          pbar.process_result(result, batch.size) if options[:verbose]
+        end
+
+        pbar.finish if options[:verbose]
+      end
+    end
+
+  private
+
+    def models
+      @models ||= begin
+                    models = options[:models] || Conf.flex_models
+                    raise ArgumentError, 'no class defined. Please use MODELS=ClassA,ClassB ' +
+                                         'or set the Flex::Configuration.flex_models properly' \
+                                         if models.nil? || models.empty?
+                    models.map{|c| c.is_a?(String) ? eval("::#{c}") : c}
+                  end
+    end
+
+  end
+
+end

data/lib/flex/refresh_callbacks.rb

@@ -0,0 +1,15 @@
+module Flex
+  module RefreshCallbacks
+
+    def self.included(base)
+      base.class_eval do
+        raise NotImplementedError, "the class #{self} must implement :after_create, :after_update and :after_destroy callbacks" \
+              unless respond_to?(:after_save) && respond_to?(:after_destroy)
+        refresh = proc{ Flex.refresh_index :index => self.class.flex.index }
+        after_save    &refresh
+        after_destroy &refresh
+      end
+    end
+
+  end
+end

data/lib/flex/result/active_model.rb

@@ -0,0 +1,61 @@
+module Flex
+  class Result
+    module ActiveModel
+
+      include Flex::Result::Scope
+
+      # extend if the context include a Flex::ActiveModel
+      def self.should_extend?(result)
+        result.variables[:context] && result.variables[:context].include?(Flex::ActiveModel)
+      end
+
+      def get_docs
+        # super is from flex-scopes
+        docs = super
+        return docs if variables[:raw_result]
+        raw_result = self
+        if docs.is_a?(Array)
+          res = docs.map {|doc| build_object(doc)}
+          res.extend(Struct::Paginable).setup(raw_result['hits']['total'], variables)
+          class << res; self end.class_eval do
+            define_method(:raw_result){ raw_result }
+            define_method(:method_missing) do |meth, *args, &block|
+              raw_result.respond_to?(meth) ? raw_result.send(meth, *args, &block) : super(meth, *args, &block)
+            end
+          end
+          res
+        else
+          build_object docs
+        end
+      end
+
+    private
+
+      def build_object(doc)
+        attrs      = (doc['_source']||{}).merge(doc['fields']||{})
+        object     = variables[:context].new attrs
+        raw_result = self
+        object.instance_eval do
+          class << self; self end.class_eval do
+            define_method(:raw_result){ raw_result }
+            define_method(:raw_document){ doc }
+            define_method(:respond_to?) do |*args|
+              doc.respond_to?(*args) || super(*args)
+            end
+            define_method(:method_missing) do |meth, *args, &block|
+              doc.respond_to?(meth) ? doc.send(meth, *args, &block) : super(meth, *args, &block)
+            end
+          end
+          @_id        = doc['_id']
+          @_version   = doc['_version']
+          @highlight  = doc['highlight']
+          # load the flex proxy before freezing
+          flex
+          self.freeze if raw_result.variables[:params][:fields] || doc['fields']
+        end
+        object
+      end
+
+    end
+  end
+end

data/lib/flex/result/document_loader.rb

@@ -0,0 +1,56 @@
+module Flex
+  class Result
+
+    #  adds sugar to documents with the following structure:
+    #
+    #    {
+    #        "_index" : "twitter",
+    #        "_type" : "tweet",
+    #        "_id" : "1",
+    #    }
+
+    module DocumentLoader
+
+      module ModelClasses
+        extend self
+        # maps all the index/types to the ruby class
+        def map
+          @map ||= begin
+                     map = {}
+                     (Conf.flex_models + Conf.flex_active_models).each do |m|
+                       m = eval("::#{m}") if m.is_a?(String)
+                       indices = m.flex.index.is_a?(Array) ? m.flex.index : [m.flex.index]
+                       types = m.flex.type.is_a?(Array) ? m.flex.type : [m.flex.type]
+                       indices.each do |i|
+                         types.each { |t| map["#{i}/#{t}"] = m }
+                       end
+                     end
+                     map
+                   end
+        end
+      end
+
+      # extend if result has a structure like a document
+      def self.should_extend?(result)
+        result.is_a? Document
+      end
+
+      def model_class
+        @model_class ||= ModelClasses.map["#{index_basename}/#{self['_type']}"]
+      end
+
+      def load
+        model_class.find(self['_id']) if model_class
+      end
+
+      def load!
+        raise DocumentMappingError, "the '#{index_basename}/#{self['_type']}' document cannot be mapped to any class." \
+              unless model_class
+        model_class.find self['_id']
+      end
+
+    end
+
+  end
+end
+

data/lib/flex/result/search_loader.rb

@@ -0,0 +1,40 @@
+module Flex
+  class Result
+    module SearchLoader
+
+      # extend if result is a Search or MultiGet
+      def self.should_extend?(result)
+        result.is_a?(Search) || result.is_a?(MultiGet)
+      end
+
+      # extend the collection on extend
+      def self.extended(result)
+        result.collection.each { |h| h.extend(DocumentLoader) }
+      end
+
+      def loaded_collection
+        @loaded_collection ||= begin
+                                 records  = []
+                                 # returns a structure like {Comment=>[{"_id"=>"123", ...}, {...}], BlogPost=>[...]}
+                                 h = Utils.group_array_by(collection) do |d|
+                                   d.model_class
+                                 end
+                                 h.each do |klass, docs|
+                                   records |= klass.find(docs.map(&:_id))
+                                 end
+                                 class_ids = collection.map { |d| [d.model_class.to_s,  d._id] }
+                                 # Reorder records to preserve order from search results
+                                 records = class_ids.map do |class_str, id|
+                                   records.detect do |record|
+                                     record.class.to_s == class_str && record.id.to_s == id.to_s
+                                   end
+                                 end
+                                 records.extend Struct::Paginable
+                                 records.setup(collection.total_entries, variables)
+                                 records
+                               end
+      end
+
+    end
+  end
+end

data/lib/flex/struct/mergeable.rb

@@ -0,0 +1,38 @@
+module Flex
+  module Struct
+    # allows deep merge between Hashes
+    module Mergeable
+
+      def deep_merge(*hashes)
+        merged = deep_dup
+        hashes.each {|h2| merged.replace(deep_merge_hash(merged,h2))}
+        merged
+      end
+
+      def deep_merge!(*hashes)
+        replace deep_merge(*hashes)
+      end
+
+      def deep_dup
+        Marshal.load(Marshal.dump(self))
+      end
+
+  private
+
+      def deep_merge_hash(h1, h2)
+        h2 ||= {}
+        h1.merge(h2) do |key, oldval, newval|
+          case
+          when oldval.is_a?(::Hash) && newval.is_a?(::Hash)
+            deep_merge_hash(oldval, newval)
+          when oldval.is_a?(::Array) && newval.is_a?(::Array)
+              oldval + newval
+          else
+            newval
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/flex-models.rb

@@ -0,0 +1,40 @@
+require 'flex'
+require 'flex-scopes'
+require 'active_attr'
+
+require 'flex/struct/mergeable'
+
+require 'flex/class_proxy/model_syncer'
+require 'flex/instance_proxy/model_syncer'
+require 'flex/model_syncer'
+
+require 'flex/class_proxy/model_indexer'
+require 'flex/instance_proxy/model_indexer'
+require 'flex/model_indexer'
+
+require 'flex/active_model/timestamps'
+require 'flex/active_model/attachment'
+require 'flex/active_model/inspection'
+require 'flex/active_model/storage'
+require 'flex/class_proxy/active_model'
+require 'flex/instance_proxy/active_model'
+require 'flex/active_model'
+
+require 'flex/refresh_callbacks'
+
+require 'flex/live_reindex_model'
+
+require 'flex/result/document_loader'
+require 'flex/result/search_loader'
+require 'flex/result/active_model'
+
+require 'flex/model_tasks'
+
+Flex::LIB_PATHS << File.dirname(__FILE__)
+
+# get_docs calls super so we make sure the result is extended by Scope first
+Flex::Conf.result_extenders  |= [ Flex::Result::DocumentLoader,
+                                  Flex::Result::SearchLoader,
+                                  Flex::Result::ActiveModel ]
+Flex::Conf.flex_models        = []
+Flex::Conf.flex_active_models = []

data/lib/tasks.rake

@@ -0,0 +1,12 @@
+require 'flex-models'
+
+env   = defined?(Rails) ? :environment : []
+
+namespace :flex do
+
+  desc 'imports from an ActiveRecord or Mongoid models'
+  task(:import => env) { Flex::ModelTasks.new.import_models }
+
+  task(:reset_redis_keys) { Flex::Redis.reset_keys }
+
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: flex-models
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Domizio Demichelis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: flex
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: flex-scopes
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: active_attr
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+description: Provides ActiveRecord, Mongoid, ActiveModel and elasticsearch-mapper-attachment
+  integrations, cross syncing, parent/child relationships, bulk-import, live-reindex
+  of models, ...
+email: dd.nexus@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- LICENSE
+- README.md
+- VERSION
+- flex-models.gemspec
+- lib/flex-models.rb
+- lib/flex/active_model.rb
+- lib/flex/active_model/attachment.rb
+- lib/flex/active_model/inspection.rb
+- lib/flex/active_model/storage.rb
+- lib/flex/active_model/timestamps.rb
+- lib/flex/class_proxy/active_model.rb
+- lib/flex/class_proxy/model_indexer.rb
+- lib/flex/class_proxy/model_syncer.rb
+- lib/flex/instance_proxy/active_model.rb
+- lib/flex/instance_proxy/model_indexer.rb
+- lib/flex/instance_proxy/model_syncer.rb
+- lib/flex/live_reindex_model.rb
+- lib/flex/model_indexer.rb
+- lib/flex/model_syncer.rb
+- lib/flex/model_tasks.rb
+- lib/flex/refresh_callbacks.rb
+- lib/flex/result/active_model.rb
+- lib/flex/result/document_loader.rb
+- lib/flex/result/search_loader.rb
+- lib/flex/struct/mergeable.rb
+- lib/tasks.rake
+homepage: http://github.com/ddnexus/flex-models
+licenses: []
+post_install_message: 
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Transparently integrates your models with one or more elasticsearch indices.
+test_files: []