flex 0.4.2 → 1.0.1

data/lib/flex/errors.rb

@@ -8,6 +8,7 @@ module Flex
   class ExistingIndexError     < StandardError; end
   class MissingHttpClientError < StandardError; end
   class MissingParentError     < StandardError; end
+  class MissingVariableError   < StandardError; end
 
   class HttpError < StandardError
 
@@ -27,7 +28,7 @@ module Flex
     end
 
     def to_s
-      log = "#{@caller_line}\n" if @caller_line
+      log = "#@caller_line\n" if @caller_line
       "#{log}#{status}: #{body}"
     end
 

data/lib/flex/http_clients/base.rb

@@ -0,0 +1,15 @@
+module Flex
+  module HttpClients
+    class Base
+
+      attr_accessor :options, :base_uri, :raise_proc
+
+      def initialize(base_uri='http://localhost:9200', options={})
+        @options    = options
+        @base_uri   = base_uri
+        @raise_proc = proc{|response| response.status >= 400}
+      end
+
+    end
+  end
+end

data/lib/flex/http_clients/loader.rb

@@ -0,0 +1,51 @@
+module Flex
+  module HttpClients
+
+    class Dummy
+      def request(*)
+        raise MissingHttpClientError,
+              'you should install the gem "patron" (recommended for performances) or "rest-client", ' +
+              'or provide your own http-client interface and set Flex::Configuration.http_client'
+      end
+    end
+
+    module Loader
+
+      extend self
+
+      def new_http_client
+        if Gem::Specification.respond_to?(:find_all_by_name)
+          case
+            # terrible way to check whether a gem is available.
+            # Gem.available? was just perfect: that's probably the reason it has been deprecated!
+            # https://github.com/rubygems/rubygems/issues/176
+          when Gem::Specification::find_all_by_name('patron').any?      then require_patron
+          when Gem::Specification::find_all_by_name('rest-client').any? then require_rest_client
+          else Dummy.new
+          end
+        else
+          case
+          when Gem.available?('patron')      then require_patron
+          when Gem.available?('rest-client') then require_rest_client
+          else Dummy.new
+          end
+        end
+      end
+
+    private
+
+      def require_patron
+        require 'patron'
+        require 'flex/http_clients/patron'
+        Patron.new
+      end
+
+      def require_rest_client
+        require 'rest-client'
+        require 'flex/http_clients/rest_client'
+        RestClient.new
+      end
+
+    end
+  end
+end

data/lib/flex/http_clients/patron.rb

@@ -1,14 +1,16 @@
 module Flex
   module HttpClients
-    module Patron
-      extend self
+    class Patron < Base
 
       def request(method, path, data=nil)
-        options = Configuration.http_client_options
-        options = options.merge(:data => data) if data
-        session.request method.to_s.downcase.to_sym, path, {}, options
+        # patron would raise an error for :post and :put requests with no data
+        # and elasticsearch ignores the data when it expects no data,
+        # so we silence patron by adding some dummy data
+        data = {} if (method == 'POST' || method == 'PUT') && data.nil?
+        opts = options.merge(:data => data)
+        session.request method.to_s.downcase.to_sym, path, {}, opts
       rescue ::Patron::TimeoutError
-        session.request method.to_s.downcase.to_sym, path, {}, options
+        session.request method.to_s.downcase.to_sym, path, {}, opts
       end
 
     private
@@ -17,7 +19,7 @@ module Flex
         Thread.current[:flex_patron_session] ||= begin
                                                    sess                       = ::Patron::Session.new
                                                    sess.headers['User-Agent'] = "flex-#{Flex::VERSION}"
-                                                   sess.base_url              = Configuration.base_uri
+                                                   sess.base_url              = base_uri
                                                    sess
                                                  end
       end

data/lib/flex/http_clients/rest_client.rb

@@ -1,15 +1,13 @@
 module Flex
   module HttpClients
-    module RestClient
-      extend self
+    class RestClient < Base
 
       def request(method, path, data=nil)
-        options = Configuration.http_client_options
-        url     = "#{Configuration.base_uri}#{path}"
-        args    = options.merge( :method  => method.to_s.downcase.to_sym,
-                                 :url     => url,
-                                 :payload => data )
-        response = ::RestClient::Request.new( args ).execute
+        url  = "#{base_uri}#{path}"
+        opts = options.merge( :method  => method.to_s.downcase.to_sym,
+                              :url     => url,
+                              :payload => data )
+        response = ::RestClient::Request.new( opts ).execute
         extend_response(response, url)
 
       rescue ::RestClient::ExceptionWithResponse => e

data/lib/flex/logger.rb

@@ -3,13 +3,34 @@ require 'logger'
 module Flex
   class Logger < ::Logger
 
+
+    attr_accessor :debug_variables, :debug_request, :debug_result, :curl_format
+
     def initialize(*)
       super
-      self.level     = ::Logger::INFO
-      self.progname  = "FLEX"
+      self.level     = ::Logger::WARN
+      self.progname  = 'FLEX'
       self.formatter = proc do |severity, datetime, progname, msg|
-        "#{msg}\n"
+        flex_format(severity, msg)
       end
+      @debug_variables = true
+      @debug_request   = true
+      @debug_result    = true
+      @curl_format     = false
+    end
+
+    def flex_format(severity, msg)
+      prefix = Dye.dye(" FLEX-#{severity} ", "FLEX-#{severity}:", :blue, :bold, :reversed) + ' '
+      msg.split("\n").map{|l| prefix + l}.join("\n") + "\n"
+    end
+
+    # force color in console (used with jruby)
+    def color=(bool)
+      Dye.color = bool
+    end
+
+    def color
+      Dye.color?
     end
 
   end

data/lib/flex/prog_bar.rb

@@ -1,7 +1,7 @@
 module Flex
   class ProgBar
 
-    attr_reader :pbar
+    attr_reader :pbar, :total_count
 
     def initialize(total_count, batch_size=nil, prefix_message=nil)
       @total_count      = total_count
@@ -18,16 +18,21 @@ module Flex
     end
 
     def process_result(result, inc)
-      Configuration.logger.error "[FLEX] Failed load:\n#{result.failed.to_yaml}" unless result.failed.size == 0
-      @failed_count     += result.failed.size
-      @successful_count += result.successful.size
+      unless result.nil? || result.empty?
+        unless result.failed.size == 0
+          Conf.logger.error "Failed load:\n#{result.failed.to_yaml}"
+          @pbar.bar_mark = 'F'
+        end
+        @failed_count     += result.failed.size
+        @successful_count += result.successful.size
+      end
       @pbar.inc(inc)
     end
 
     def finish
       @pbar.finish
-      puts "Processed #{@total_count}. Successful #{@successful_count}. Skipped #{@total_count - @successful_count - @failed_count}. Failed #{@failed_count}."
-      puts "See the log for the details about the failed import." unless @failed_count == 0
+      puts "Processed #@total_count. Successful #@successful_count. Skipped #{@total_count - @successful_count - @failed_count}. Failed #@failed_count."
+      puts 'See the log for the details about the failures.' unless @failed_count == 0
     end
 
   end

data/lib/flex/rails.rb

@@ -1,13 +1 @@
-require 'flex'
-require 'rails'
-require 'flex/rails/helper'
-
-if ::Rails.respond_to?(:version) && ::Rails.version.to_i >= 3
-  require 'flex/rails/engine'
-else
-  Flex::Configuration.configure do |c|
-    c.config_file = ::Rails.root.join('config', 'flex.yml').to_s
-    c.flex_dir    = ::Rails.root.join('app', 'flex').to_s
-    c.debug       = ::Rails.env.development?
-  end
-end
+raise NotImplementedError, %(The "flex/rails" file has been replaced by the "flex-rails" gem. Please, use "flex-rails" in place of "flex" in Rails applications (change "gem 'flex', require => 'flex/rails'" to "gem 'flex-rails'" in your Gemfile))

data/lib/flex/result.rb

@@ -1,18 +1,24 @@
 module Flex
   class Result < ::Hash
 
-    attr_reader :template, :variables, :response
+    attr_reader :template, :response
+    attr_accessor :variables
 
     def initialize(template, variables, response, result=nil)
       @template  = template
       @variables = variables
       @response  = response
       replace result || !response.body.empty? && MultiJson.decode(response.body) || return
-      Configuration.result_extenders.each do |ext|
+      Conf.result_extenders.each do |ext|
         next if ext.respond_to?(:should_extend?) && !ext.should_extend?(self)
         extend ext
       end
     end
 
+    def to_flex_result(force=false)
+      return self if variables[:context].nil? || variables[:raw_result] &&! force
+      variables[:context].respond_to?(:flex_result) ? variables[:context].flex_result(self) : self
+    end
+
   end
 end

data/lib/flex/result/document.rb

@@ -1,36 +1,65 @@
 module Flex
   class Result
 
-    #  adds sugar to documents with the following structure:
+    #  adds sugar to documents with the following structure (_source is optional):
     #
     #    {
     #        "_index" : "twitter",
     #        "_type" : "tweet",
     #        "_id" : "1",
+    #        "_source" : {
+    #            "user" : "kimchy",
+    #            "postDate" : "2009-11-15T14:12:12",
+    #            "message" : "trying out Elastic Search"
+    #        }
     #    }
 
     module Document
 
-      META = %w[_index _type _id]
-
       # extend if result has a structure like a document
       def self.should_extend?(obj)
-        META.all? {|k| obj.has_key?(k)}
+        %w[_index _type _id].all? {|k| obj.has_key?(k)}
+      end
+
+      def respond_to?(meth, private=false)
+        smeth = meth.to_s
+        readers.has_key?(smeth) || has_key?(smeth) || has_key?("_#{smeth}") || super
       end
 
-      META.each do |m|
-        define_method(m){self["#{m}"]}
+      # exposes _source and readers: automatically supply object-like reader access
+      # also expose meta readers like _id, _source, etc, also callable without the leading '_'
+      def method_missing(meth, *args, &block)
+        smeth = meth.to_s
+        case
+        # field name
+        when readers.has_key?(smeth)
+          readers[smeth]
+        # result item
+        when has_key?(smeth)
+          self[smeth]
+        # result item called without the '_' prefix
+        when has_key?("_#{smeth}")
+          self["_#{smeth}"]
+        else
+          super
+        end
       end
 
-      def mapped_class(should_raise=false)
-        @mapped_class ||= Manager.type_class_map["#{_index}/#{_type}"]
-      rescue NameError
-        raise DocumentMappingError, "the '#{_index}/#{_type}' document cannot be mapped to any class." \
-              if should_raise
+      # used to get the unprefixed (by live-reindex) index name
+      def index_basename
+        @index_basename ||= self['_index'].sub(/^\d{14}_/, '')
       end
 
-      def load
-        mapped_class.find _id
+
+      private
+
+      def readers
+        @readers ||= begin
+                       readers = (self['_source']||{}).merge(self['fields']||{})
+                       # flattened reader for multi_readers or attachment readers
+                       readers.keys.each{|k| readers[k.gsub('.','_')] = readers.delete(k) if k.include?('.')}
+                       readers
+                     end
       end
 
     end

data/lib/flex/result/multi_get.rb

@@ -0,0 +1,24 @@
+module Flex
+  class Result
+    module MultiGet
+
+      # extend if result comes from a search url
+      def self.should_extend?(result)
+        result.response.url =~ /\b_mget\b/ && result['docs']
+      end
+
+      # extend the hits results on extended
+      def self.extended(result)
+        result['docs'].each { |h| h.extend(Document) }
+        result['docs'].extend Struct::Paginable
+        result['docs'].setup(result['docs'].size, result.variables)
+      end
+
+      def docs
+        self['docs']
+      end
+      alias_method :collection, :docs
+
+    end
+  end
+end

data/lib/flex/result/search.rb

@@ -10,7 +10,7 @@ module Flex
       # extend the hits results on extended
       def self.extended(result)
         result['hits']['hits'].each { |h| h.extend(Document) }
-        result['hits']['hits'].extend Collection
+        result['hits']['hits'].extend Struct::Paginable
         result['hits']['hits'].setup(result['hits']['total'], result.variables)
       end
 
@@ -23,29 +23,6 @@ module Flex
         self['facets']
       end
 
-      def loaded_collection
-        @loaded_collection ||= begin
-                                 records  = []
-                                 # returns a structure like {Comment=>[{"_id"=>"123", ...}, {...}], BlogPost=>[...]}
-                                 h = Utils.group_array_by(collection) do |d|
-                                       d.mapped_class(should_raise=true)
-                                     end
-                                 h.each do |klass, docs|
-                                   records |= klass.find(docs.map(&:_id))
-                                 end
-                                 class_ids = collection.map { |d| [d.mapped_class.to_s,  d._id] }
-                                 # Reorder records to preserve order from search results
-                                 records = class_ids.map do |class_str, id|
-                                             records.detect do |record|
-                                               record.class.to_s == class_str && record.id.to_s == id.to_s
-                                             end
-                                           end
-                                records.extend Collection
-                                records.setup(self['hits']['total'], variables)
-                                records
-                              end
-      end
-
     end
   end
 end

data/lib/flex/struct/array.rb

@@ -0,0 +1,25 @@
+module Flex
+  module Struct
+    class Array < ::Array
+
+      include Symbolize
+
+      def push(*vals)
+        super *symbolize(vals)
+      end
+
+      def <<(val)
+        super symbolize(val)
+      end
+
+      def insert(*vals)
+        super *symbolize(vals)
+      end
+
+      def unshift(*vals)
+        super *symbolize(vals)
+      end
+
+    end
+  end
+end

data/lib/flex/struct/hash.rb

@@ -0,0 +1,105 @@
+module Flex
+  module Struct
+    class Hash < ::Hash
+      include Symbolize
+
+      def initialize
+        super do |hash, key|
+          if key[-1] == '!'
+            klass = (key[0] == '_' ? Array : Hash)
+            hash[clean_key(key)] = klass.new
+          end
+        end
+      end
+
+      def merge(hash)
+        super symbolize(hash)
+      end
+
+      def merge!(hash)
+        super symbolize(hash)
+      end
+
+      def store(key, val)
+        if key[-1] == '='
+          super key[0..-2].to_sym, val.extend(AsIs)
+        else
+          super clean_key(key), symbolize(val)
+        end
+      end
+      alias_method :[]=, :store
+
+      def fetch(key, *rest, &block)
+        cleaned = clean_key(key)
+        super has_key?(cleaned) ? cleaned : key.to_sym, *rest, &block
+      end
+
+      def [](key)
+        cleaned = clean_key(key)
+        super has_key?(cleaned) ? cleaned : key.to_sym
+      end
+
+      def deep_merge(*hashes)
+        dupe = deep_dup(self)
+        hashes.each {|h2| dupe.replace(deep_merge_hash(dupe,h2))}
+        dupe
+      end
+
+      def deep_merge!(*hashes)
+        replace deep_merge(*hashes)
+      end
+
+      module Nil
+        def method_missing(*)
+          self
+        end
+      end
+
+      def try(key)
+        has_key?(key) ? self[key] : nil.extend(Nil)
+      end
+
+      def try_delete(key, *rest, &block)
+        val = delete clean_key(key), *rest, &block
+        val.nil? ? nil.extend(Nil) : val
+      end
+
+
+      private
+
+      def clean_key(key)
+        key[-1] == '!' ? key[0..-2].to_sym : key.to_sym
+      end
+
+      def deep_merge_hash(h1, h2)
+        h2 ||= {}
+        h1.merge(h2) do |key, oldval, newval|
+          case
+          when oldval.is_a?(Hash) && newval.is_a?(Hash)
+            deep_merge_hash(oldval, newval)
+          when oldval.is_a?(Array) && newval.is_a?(Array)
+            oldval | newval
+          else
+            newval
+          end
+        end
+      end
+
+      def deep_dup(obj)
+        case obj
+        when ::Hash, Flex::Struct::Hash
+          h = obj.dup
+          h.each_pair do |k,v|
+            h[k] = deep_dup(v)
+          end
+          h
+        when ::Array, Flex::Struct::Array
+          obj.map{|i| deep_dup(i)}
+        else
+          obj
+        end
+      end
+
+    end
+  end
+end