flex 0.1.0

data/lib/flex/result/document.rb

@@ -0,0 +1,38 @@
+module Flex
+  class Result
+
+    #  adds sugar to documents with the following structure:
+    #
+    #    {
+    #        "_index" : "twitter",
+    #        "_type" : "tweet",
+    #        "_id" : "1",
+    #    }
+
+    module Document
+
+      META = %w[_index _type _id]
+
+      # extend if result has a structure like a document
+      def self.should_extend?(obj)
+        META.all? {|k| obj.has_key?(k)}
+      end
+
+      META.each do |m|
+        define_method(m){self["#{m}"]}
+      end
+
+      def mapped_class(should_raise=false)
+        @mapped_class ||= Model::Manager.type_class_map["#{_index}/#{_type}"]
+      rescue NameError
+        raise DocumentMappingError, "the '#{_index}/#{_type}' document cannot be mapped to any class." \
+              if should_raise
+      end
+
+      def load
+        mapped_class.find _id
+      end
+
+    end
+  end
+end

data/lib/flex/result/indifferent_access.rb

@@ -0,0 +1,11 @@
+module Flex
+  module Result
+    module IndifferentAccess
+
+      def self.extended(result)
+        result.extend Structure::IndifferentAccess
+      end
+
+    end
+  end
+end

data/lib/flex/result/search.rb

@@ -0,0 +1,51 @@
+module Flex
+  class Result
+    module Search
+
+      # extend if result comes from a search url
+      def self.should_extend?(result)
+        result.response.url =~ /\b_m?search\b/ && result['hits']
+      end
+
+      # extend the hits results on extended
+      def self.extended(result)
+        result['hits']['hits'].each { |h| h.extend(Document) }
+        result['hits']['hits'].extend Collection
+        result['hits']['hits'].setup(result['hits']['total'], result.variables)
+      end
+
+      def collection
+        self['hits']['hits']
+      end
+      alias_method :documents, :collection
+
+      def facets
+        self['facets']
+      end
+
+      def loaded_collection
+        @loaded_collection ||= begin
+                                 records  = []
+                                 # returns a structure like {Comment=>[{"_id"=>"123", ...}, {...}], BlogPost=>[...]}
+                                 h = Utils.group_array_by(collection) do |d|
+                                       d.mapped_class(should_raise=true)
+                                     end
+                                 h.each do |klass, docs|
+                                   records |= klass.find(docs.map(&:_id))
+                                 end
+                                 class_ids = collection.map { |d| [d.mapped_class.to_s,  d._id] }
+                                 # Reorder records to preserve order from search results
+                                 records = class_ids.map do |class_str, id|
+                                             records.detect do |record|
+                                               record.class.to_s == class_str && record.id.to_s == id.to_s
+                                             end
+                                           end
+                                records.extend Collection
+                                records.setup(self['hits']['total'], variables)
+                                records
+                              end
+      end
+
+    end
+  end
+end

data/lib/flex/result/source_document.rb

@@ -0,0 +1,63 @@
+module Flex
+  class Result
+
+    #  adds sugar to documents with the following structure:
+    #
+    #    {
+    #        "_index" : "twitter",
+    #        "_type" : "tweet",
+    #        "_id" : "1",
+    #        "_source" : {
+    #            "user" : "kimchy",
+    #            "postDate" : "2009-11-15T14:12:12",
+    #            "message" : "trying out Elastic Search"
+    #        }
+    #    }
+
+    module SourceDocument
+
+      # extend if result has a structure like a document
+      def self.should_extend?(obj)
+        %w[_index _type _id _source].all? {|k| obj.has_key?(k)}
+      end
+
+      # exposes _source: automatically supply object-like reader access
+      # also expose meta fields like _id, _source, etc, also for methods without the leading '_'
+      def method_missing(meth, *args, &block)
+        case
+        when meth.to_s =~ /^_/ && has_key?(meth.to_s) then self[meth.to_s]
+        when self['_source'].has_key?(meth.to_s)      then self['_source'][meth.to_s]
+        when has_key?("_#{meth.to_s}")                then self["_#{meth.to_s}"]
+        else super
+        end
+      end
+
+      # returns the _source hash with an added id (if missing))
+      def to_attributes
+        {'id' => _id}.merge(_source)
+      end
+
+      # creates an instance of a mapped or computed class, falling back to OpenStruct
+      def to_mapped
+        to(mapped_class || OpenStruct)
+      end
+
+      # experimental: creates an instance of klass out of to_attributes
+      # we should probably reset the id to the original document _id
+      # but be sure the record is read-only
+      def to(klass)
+        obj = klass.new(to_attributes)
+        case
+        when defined?(ActiveRecord::Base) && obj.is_a?(ActiveRecord::Base)
+          obj.readonly!
+        when defined?(Mongoid::Document)  && obj.is_a?(Mongoid::Document)
+          # TODO: make it readonly
+        when obj.is_a?(OpenStruct)
+          # TODO: anythig to extend?
+        end
+        obj
+      end
+
+    end
+  end
+end

data/lib/flex/result/source_search.rb

@@ -0,0 +1,32 @@
+module Flex
+  class Result
+    module SourceSearch
+
+      # extend if result comes from a search url and does not contain an empty fields param (no _source))
+      def self.should_extend?(result)
+        result.response.url =~ /\b_m?search\b/ &&
+            !result['hits']['hits'].empty? && result['hits']['hits'].first.has_key?('_source')
+      end
+
+      # extend the hits results on extended
+      def self.extended(result)
+        result['hits']['hits'].each { |h| h.extend(SourceDocument) }
+      end
+
+      # experimental
+      # returns an array of document mapped objects
+      def mapped_collection
+        @mapped_collection ||= begin
+                               docs = self['hits']['hits'].map do |h|
+                                        raise NameError, "no '_source' found in hit #{h.inspect} " \
+                                              unless h.respond_to(:to_mapped)
+                                        h.to_mapped
+                                      end
+                               docs.extend Collection
+                               docs.setup(self['hits']['total'], variables)
+                             end
+      end
+
+    end
+  end
+end

data/lib/flex/structure/indifferent_access.rb

@@ -0,0 +1,44 @@
+module Flex
+  module Structure
+    # allows to use both Symbol or String keys to access the same values in a Hash
+    module IndifferentAccess
+
+      def [](k)
+        get_value(k)
+      end
+
+      def []=(k,v)
+        # default to_s for storing new keys
+        has_key?(k) ? super : super(k.to_s, v)
+      end
+
+      def to_hash
+        self
+      end
+
+    private
+
+      def get_value(k)
+        val = fetch_val(k)
+        case val
+        when Hash
+          val.extend IndifferentAccess
+        when Array
+          val.each {|v| v.extend IndifferentAccess if v.is_a?(Hash)}
+        end
+        val
+      end
+
+      def fetch_val(k)
+        v = fetch(k, nil)
+        return v unless v.nil?
+        if k.is_a?(String)
+          v = fetch(k.to_sym, nil)
+          return v unless v.nil?
+        end
+        fetch(k.to_s, nil) if k.is_a?(Symbol)
+      end
+
+    end
+  end
+end

data/lib/flex/structure/mergeable.rb

@@ -0,0 +1,21 @@
+module Flex
+  module Structure
+    # allows deep merge between Hashes
+    module Mergeable
+
+      def deep_merge(*hashes)
+        Utils.deep_merge_hashes(self, *hashes)
+      end
+
+      def deep_merge!(*hashes)
+        replace deep_merge(*hashes)
+      end
+      alias_method :add, :deep_merge!
+
+      def deep_dup
+        Marshal.load(Marshal.dump(self))
+      end
+
+    end
+  end
+end

data/lib/flex/tasks.rb

@@ -0,0 +1,141 @@
+module Flex
+  module Tasks
+    extend self
+
+    def create_indices
+      indices.each do |index|
+        delete_index(index) if ENV['FORCE']
+        raise ExistingIndexError, "#{index.inspect} already exists. Please use FORCE=1 if you want to delete it first." \
+              if exist?(index)
+        create(index)
+      end
+    end
+
+    def delete_indices
+      indices.each { |index| delete_index(index) }
+    end
+
+    def import_models
+      require 'progressbar'
+
+      Configuration.http_client_options[:timeout]   = ENV['TIMEOUT'].to_i if ENV['TIMEOUT']
+      Configuration.http_client_options[:timeout] ||= 20
+      Configuration.debug = !!ENV['FLEX_DEBUG']
+      batch_size = ENV['BATCH_SIZE'] && ENV['BATCH_SIZE'].to_i || 1000
+      options = {}
+      if ENV['IMPORT_OPTIONS']
+        ENV['IMPORT_OPTIONS'].split('&').each do |pair|
+          k, v  = pair.split('=')
+          options[k.to_sym] = v
+        end
+      end
+      deleted    = []
+
+      models.each do |klass|
+        index = klass.flex.index
+
+        if ENV['FORCE']
+          unless deleted.include?(index)
+            delete_index(index)
+            deleted << index
+            puts "#{index} index deleted"
+          end
+        end
+
+        unless exist?(index)
+          create(index)
+          puts "#{index} index created"
+        end
+
+        if defined?(Mongoid::Document) && klass.ancestors.include?(Mongoid::Document)
+          def klass.find_in_batches(options={})
+            0.step(count, options[:batch_size]) do |offset|
+              yield limit(options[:batch_size]).skip(offset).to_a
+            end
+          end
+        end
+
+        unless klass.respond_to?(:find_in_batches)
+          STDERR.puts "[ERROR] Class #{klass} does not respond to :find_in_batches. Skipped."
+          next
+        end
+
+        total_count      = klass.count
+        successful_count = 0
+        failed           = []
+
+        pbar = ProgressBar.new('processing...', total_count)
+        pbar.clear
+        pbar.bar_mark = '|'
+        puts '_' * pbar.send(:get_term_width)
+        puts "Class #{klass}: indexing #{total_count} documents in batches of #{batch_size}:\n"
+        pbar.send(:show)
+
+        klass.find_in_batches(:batch_size => batch_size) do |array|
+          opts = {:index => index}.merge(options)
+          result = Flex.import_collection(array, opts) || next
+          f = result.failed
+          failed += f
+          successful_count += result.successful.count
+          pbar.inc(array.size)
+        end
+
+        pbar.finish
+        puts "Processed #{total_count}. Successful #{successful_count}. Skipped #{total_count - successful_count - failed.size}. Failed #{failed.size}."
+
+        if failed.size > 0
+          puts 'Failed imports:'
+          puts failed.to_yaml
+        end
+      end
+    end
+
+  private
+
+    def indices
+      indices = ENV['INDEX'] || ENV['INDICES'] || struct.keys
+      indices = eval(indices) if indices.is_a?(String)
+      indices = [indices] unless indices.is_a?(Array)
+      indices
+    end
+
+    def exist?(index)
+      Flex.exist?(:index => index)
+    end
+
+    def struct
+      @struct ||= begin
+                    @indices_yaml = ENV['CONFIG_FILE'] || Flex::Configuration.config_file
+                    raise Errno::ENOENT, "no such file or directory #{@indices_yaml.inspect}. " +
+                                         'Please, use CONFIG_FILE=/path/to/index.yml ' +
+                                         'or set the Flex::Configuration.config_file properly' \
+                          unless File.exist?(@indices_yaml)
+                    Model::Manager.indices(@indices_yaml)
+                  end
+    end
+
+
+    def models
+      @models ||= begin
+        mods = ENV['MODEL'] || ENV['MODELS'] || Flex::Configuration.flex_models
+        raise AgrumentError, 'no class defined. Please use MODEL=AClass or MODELS=[ClassA,ClassB]' +
+                             'or set the Flex::Configuration.flex_models properly' \
+              if mods.nil? || mods.empty?
+        mods = eval(mods) if mods.is_a?(String)
+        mods = [mods] unless mods.is_a?(Array)
+        mods.map{|c| c.is_a?(String) ? eval("::#{c}") : c}
+      end
+    end
+
+    def delete_index(index)
+      Flex.delete_index(:index => index) if exist?(index)
+    end
+
+    def create(index)
+      raise MissingIndexEntryError, "no #{index.inspect} entry defined in #@indices_yaml" \
+            unless struct.has_key?(index)
+      Flex.POST "/#{index}", struct[index]
+    end
+
+  end
+end

data/lib/flex/template.rb

@@ -0,0 +1,187 @@
+module Flex
+  # Generic Flex::Template object.
+  # This class represents a generic Flex::Template object. It is used as the base class by all the more specific Flex::Template::* classes.
+  # You  usually don't need to instantiate this class manually, since it is usually used internally.
+  # For more details about templates, see the documentation.
+  class Template
+
+    include Base
+
+    def self.variables
+      Variables.new
+    end
+
+    attr_reader :method, :path, :data, :variables, :tags, :partials
+
+    def initialize(method, path, data=nil, vars=nil)
+      @method = method.to_s.upcase
+      raise ArgumentError, "#@method method not supported" \
+            unless %w[HEAD GET PUT POST DELETE].include?(@method)
+      @path          = path =~ /^\// ? path : "/#{path}"
+      @data          = Utils.data_from_source(data)
+      @instance_vars = vars
+    end
+
+    def setup(host_flex, name=nil, source_vars=nil)
+      @host_flex   = host_flex
+      @name        = name
+      @source_vars = source_vars
+      self
+    end
+
+    def render(vars={})
+      do_render(vars) do |response, int|
+        Result.new(self, int[:vars], response)
+      end
+    end
+
+    def to_a(vars={})
+      int = interpolate(vars)
+      a = [method, int[:path], int[:data], @instance_vars]
+      2.times { a.pop if a.last.nil? }
+      a
+    end
+
+    def to_curl(vars={})
+      to_curl_string interpolate(vars, strict=true)
+    end
+
+    def to_flex(name=nil)
+      (name ? {name.to_s => to_a} : to_a).to_yaml
+    end
+
+  private
+
+    def do_render(vars={})
+      int          = interpolate(vars, strict=true)
+      path         = build_path(int, vars)
+      encoded_data = build_data(int, vars)
+      response     = Configuration.http_client.request(method, path, encoded_data)
+
+      # used in Flex.exist?
+      return response.status == 200 if method == 'HEAD'
+
+      if Configuration.raise_proc.call(response)
+        int[:vars][:raise].is_a?(FalseClass) ? return : raise(HttpError.new(response, caller_line))
+      end
+
+      result = yield(response, int)
+
+    rescue NameError => e
+      if e.name == :request
+        raise MissingHttpClientError, 'you should install the gem "patron" (recommended for performances) or "rest-client", ' +
+                                      'or provide your own http-client interface and set Flex::Configuration.http_client'
+      else
+        raise
+      end
+    ensure
+      to_logger(path, encoded_data, result) if int && Configuration.debug && Configuration.logger.level == 0
+    end
+
+    def build_path(int, vars)
+      params = int[:vars][:params]
+      path   = vars[:path] || int[:path]
+      if params
+        path << ((path =~ /\?/) ? '&' : '?')
+        path << params.map { |p| p.join('=') }.join('&')
+      end
+      path
+    end
+
+    def build_data(int, vars)
+      data = vars[:data] && Utils.data_from_source(vars[:data]) || int[:data]
+      (data.nil? || data.is_a?(String)) ? data : MultiJson.encode(data)
+    end
+
+    def to_logger(path, encoded_data, result)
+      h = {}
+      h[:method] = method
+      h[:path]   = path
+      h[:data]   = MultiJson.decode(encoded_data) unless encoded_data.nil?
+      h[:result] = result if result && Configuration.debug_result
+      log        = Configuration.debug_to_curl ? to_curl_string(h) : Utils.stringified_hash(h).to_yaml
+      Configuration.logger.debug "[FLEX] Rendered #{caller_line}\n#{log}"
+    end
+
+    def caller_line
+      method_name = @host_flex && @name && "#{@host_flex.host_class}.#@name"
+      line = caller.find{|l| l !~ %r|/flex/lib/flex/[^\.]+\.rb|}
+      ll = ''
+      ll << "#{method_name} from " if method_name
+      ll << "#{line}"
+      ll
+    end
+
+    def to_curl_string(h)
+      pretty = h[:path] =~ /\?/ ? '&pretty=1' : '?pretty=1'
+      curl =  %(curl -X#{method} "#{Configuration.base_uri}#{h[:path]}#{pretty}")
+      if h[:data]
+        data = if h[:data].is_a?(String)
+                 h[:data].length > 1024 ? h[:data][0,1024] + '...(truncated display)' : h[:data]
+               else
+                 MultiJson.encode(h[:data], :pretty => true)
+               end
+        curl << %( -d '\n#{data}\n')
+      end
+      curl
+    end
+
+    def interpolate(*args)
+      tags        = Tags.new
+      stringified = tags.stringify(:path => @path, :data => @data)
+      @partials, @tags = tags.map(&:name).partition{|n| n.to_s =~ /^_/}
+      @variables = Configuration.variables.deep_dup
+      @variables.add(self.class.variables)
+      @variables.add(@host_flex.variables) if @host_flex
+      @variables.add(@source_vars, @instance_vars, tags.variables)
+      instance_eval <<-ruby, __FILE__, __LINE__
+        def interpolate(vars={}, strict=false)
+          return {:path => path, :data => data, :vars => vars} if vars.empty? && !strict
+          sym_vars = {}
+          vars.each{|k,v| sym_vars[k.to_sym] = v} # so you can pass the rails params hash
+          merged = @variables.deep_merge sym_vars
+          vars   = process_vars(merged)
+          obj    = #{stringified}
+          obj[:vars] = vars
+          obj    = prune(obj, vars[:no_pruning])
+          obj
+        end
+      ruby
+      interpolate(*args)
+    end
+
+    # prunes the branch when the leaf is nil
+    # also removes empty path segments
+    # and compact.flatten the Array values
+    def prune(obj, no_pruning)
+      case obj
+      when Array
+        return if obj.empty?
+        a = obj.map do |i|
+              next if i.nil?
+              prune(i, no_pruning)
+            end.compact.flatten
+        a unless a.empty?
+      when Hash
+        return if obj.empty?
+        h = {}
+        obj.each do |k, v|
+          next if v.nil?
+          v = case
+              when k == :path
+                v.gsub(/\/+/, '/')
+              when no_pruning.include?(k)
+                v
+              else
+                prune(v, no_pruning)
+              end
+          h[k] = v unless v.nil?
+        end
+        h unless h.empty?
+      else
+        obj
+      end
+    end
+
+  end
+end