flex 0.4.2 → 1.0.1

data/lib/flex/utils.rb

@@ -2,23 +2,17 @@ module Flex
   module Utils
     extend self
 
-    def data_from_source(source)
+    def parse_source(source)
       return unless source
-      data = case source
-             when Hash              then stringified_hash(source)
-             when /^\s*\{.+\}\s*$/m then source
-             when String            then YAML.load(source)
-             else raise ArgumentError, "expected a String or Hash instance (got #{source.inspect})"
-             end
-      raise ArgumentError, "the source does not decode to a Hash or String (got #{data.inspect})" \
-            unless data.is_a?(Hash) || data.is_a?(String)
-      data
-    end
-
-    def deep_merge_hashes(h1, *hashes)
-      merged = h1.dup
-      hashes.each {|h2| merged.replace(deep_merge_hash(merged,h2))}
-      merged
+      parsed = case source
+               when Hash              then keyfy(:to_s, source)
+               when /^\s*\{.+\}\s*$/m then source
+               when String            then YAML.load(source)
+               else raise ArgumentError, "expected a String or Hash instance, got #{source.inspect}"
+               end
+      raise ArgumentError, "the source does not decode to an Array, Hash or String, got #{parsed.inspect}" \
+            unless parsed.is_a?(Hash) || parsed.is_a?(Array) || parsed.is_a?(String)
+      parsed
     end
 
     def erb_process(source)
@@ -39,22 +33,71 @@ module Flex
       h
     end
 
-    def stringified_hash(hash)
-      h = {}
-      hash.each do |k,v|
-        h[k.to_s] = v.is_a?(Hash) ? stringified_hash(v) : v
+    def delete_allcaps_keys(hash)
+      hash.keys.each { |k| hash.delete(k) if k =~ /^[A-Z_]+$/ }
+      hash
+    end
+
+    def keyfy(to_what, obj)
+      case obj
+      when Hash
+        h = {}
+        obj.each do |k,v|
+          h[k.send(to_what)] = keyfy(to_what, v)
+        end
+        h
+      when Array
+        obj.map{|i| keyfy(to_what, i)}
+      else
+        obj
       end
+    end
+
+    def slice_hash(hash, *keys)
+      h = {}
+      keys.each{|k| h[k] = hash[k] if hash.has_key?(k)}
       h
     end
 
-  private
+    def env2options(*keys)
+      options = {}
+      ENV.keys.map do |k|
+        key = k.downcase.to_sym
+        options[key] = ENV[k] if keys.include?(key)
+      end
+      options
+    end
+
+    def define_delegation(opts)
+      file, line = caller.first.split(':', 2)
+      line = line.to_i
+
+      obj, meth, methods, to = opts[:in], opts[:by], opts[:for], opts[:to]
 
-    def deep_merge_hash(h1, h2)
-      h2 ||= {}
-      h1.merge(h2) do |key, oldval, newval|
-        oldval.is_a?(Hash) && newval.is_a?(Hash) ? deep_merge_hash(oldval, newval) : newval
+      methods.each do |method|
+        obj.send meth, <<-method, file, line - 1
+          def #{method}(*args, &block)                        # def method_name(*args, &block)
+            if #{to} || #{to}.respond_to?(:#{method})         #   if client || client.respond_to?(:name)
+              #{to}.__send__(:#{method}, *args, &block)       #     client.__send__(:name, *args, &block)
+            end                                               #   end
+          end                                                 # end
+        method
       end
+
     end
 
+    def class_name_to_type(class_name)
+      type = class_name.tr(':', '_')
+      type.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1_\2')
+      type.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
+      type.downcase!
+      type
+    end
+
+    def type_to_class_name(type)
+      type.gsub(/__(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+    end
+
+
   end
 end

data/lib/flex/variables.rb

@@ -1,11 +1,62 @@
 module Flex
-  class Variables < Hash
+  class Variables < Struct::Hash
 
-    include Structure::Mergeable
+    def initialize(*hashes)
+      deep_merge! super(), *hashes
+    end
+
+    def finalize
+      self[:index] = self[:index].uniq.join(',') if self[:index].is_a?(Array)
+      self[:type]  = self[:type].uniq.join(',')  if self[:type].is_a?(Array)
+      # so you can pass :fields => [:field_one, :field_two]
+      self[:params!].each{|k,v| self[:params][k] = v.uniq.join(',') if v.is_a?(Array)}
+      if self[:page]
+        self[:page] = self[:page].to_i
+        self[:page] = 1 unless self[:page] > 0
+        self[:params][:from] ||= ((self[:page] - 1) * (self[:params][:size] || 10)).ceil unless self[:page] == 1
+      else
+        self[:page] = 1
+      end
+      self
+    end
+
+    # returns Prunable::Value if the value is in VALUES (called from stringified)
+    def get_prunable(key)
+      val = fetch_nested(key)
+      return val if self[:no_pruning].include?(key)
+      Prunable::VALUES.include?(val) ? Prunable::Value : val
+    end
+
+    # allows to store keys like 'a.3.c' into vars[:a][3][:c]
+    def store_nested(key, value)
+      var = unnest(key).reverse.inject(value) do |memo,k|
+              if k.is_a?(Symbol)
+                {k => memo}
+              else
+                ar = []
+                ar[k] = memo
+                ar
+              end
+            end
+      deep_merge! var
+    end
+
+    # allows to fetch values for tag names like 'a.3.c' fetching vars[:a][3][:c]
+    def fetch_nested(key)
+      unnest(key).inject(self, :fetch)
+    rescue NoMethodError, KeyError
+      raise MissingVariableError, "the required #{key.inspect} variable is missing."
+    end
 
-    def initialize(hash={})
-      replace hash
+    private
+
+    def unnest(key)
+      key.to_s.split('.').map{|s| s =~ /^\d+$/ ? s.to_i : s.to_sym}
     end
 
   end
+  # shorter alias
+  Vars = Variables
 end
+
+

data/lib/tasks.rake

@@ -0,0 +1,28 @@
+require 'flex'
+
+env   = defined?(Rails) ? :environment : []
+
+namespace :flex do
+
+  # deprecated tasks
+  task(:create_indices => env) do
+    Flex::Deprecation.warn 'flex:create_indices', 'flex:index:create', nil
+    Flex::Tasks.new.create_indices
+  end
+  task(:delete_indices => env) do
+    Flex::Deprecation.warn 'flex:delete_indices', 'flex:index:delete', nil
+    Flex::Tasks.new.delete_indices
+  end
+
+  namespace :index do
+
+    desc 'creates index/indices from the Flex::Configuration.config_file file'
+    task(:create => env) { Flex::Tasks.new.create_indices }
+
+    desc 'destroys index/indices in the Flex::Configuration.config_file file'
+    task(:delete => env) { Flex::Tasks.new.delete_indices }
+
+  end
+
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flex
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 1.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,83 +9,78 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-23 00:00:00.000000000 Z
+date: 2013-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70261802510500 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.3.4
   type: :runtime
   prerelease: false
-  version_requirements: *70261802510500
-- !ruby/object:Gem::Dependency
-  name: progressbar
-  requirement: &70261802508900 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.11.0
-  type: :runtime
-  prerelease: false
-  version_requirements: *70261802508900
+        version: 1.3.4
 - !ruby/object:Gem::Dependency
-  name: prompter
-  requirement: &70261802507520 !ruby/object:Gem::Requirement
+  name: progressbar
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.11.0
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.5
+        version: 0.12.0
   type: :runtime
   prerelease: false
-  version_requirements: *70261802507520
-- !ruby/object:Gem::Dependency
-  name: irt
-  requirement: &70261802506900 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.11.0
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2.10
-  type: :development
-  prerelease: false
-  version_requirements: *70261802506900
+        version: 0.12.0
 - !ruby/object:Gem::Dependency
-  name: patron
-  requirement: &70261802506120 !ruby/object:Gem::Requirement
+  name: dye
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.4.18
-  type: :development
+        version: 0.1.4
+  type: :runtime
   prerelease: false
-  version_requirements: *70261802506120
-- !ruby/object:Gem::Dependency
-  name: rest-client
-  requirement: &70261802505360 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.6.7
-  type: :development
-  prerelease: false
-  version_requirements: *70261802505360
-description: ! 'Flex is an innovative ruby client for ElasticSearch. With Flex your
-  code will be clean, easy to write and read, and very short: "poetic-short". Flex
-  is fast and efficient, easy to use and customize, and offers ActiveRecord, Mongoid
-  and Rails integration.
+        version: 0.1.4
+description: ! 'Flex is the ultimate ruby client for elasticsearch. It is powerful,
+  fast and efficient, easy to use and customize.
+
+
+  It covers ALL the elasticsearch API, and transparently integrates it with your app
+  and its components, like Rails, ActiveRecord, Mongoid, ActiveModel, will_paginate,
+  kaminari, elasticsearch-mapper-attachments, ...
+
+
+  It also implements and integrates very advanced features like chainable scopes,
+  live-reindex, cross-model syncing, query fragment reuse, parent/child relationships,
+  templating, self-documenting tools, detailed debugging, ...
 
 '
 email: dd.nexus@gmail.com
-executables:
-- flexes
+executables: []
 extensions: []
 extra_rdoc_files:
 - README.md
@@ -93,72 +88,54 @@ files:
 - LICENSE
 - README.md
 - VERSION
-- bin/flexes
 - flex.gemspec
 - lib/flex.rb
-- lib/flex/api_methods.yml
+- lib/flex/api_stubs.rb
+- lib/flex/api_templates/cluster_api.yml
+- lib/flex/api_templates/core_api.yml
+- lib/flex/api_templates/indices_api.yml
 - lib/flex/class_proxy/base.rb
-- lib/flex/class_proxy/loader.rb
-- lib/flex/class_proxy/model.rb
-- lib/flex/class_proxy/model_sync.rb
-- lib/flex/class_proxy/related_model.rb
+- lib/flex/class_proxy/templates.rb
+- lib/flex/class_proxy/templates/doc.rb
+- lib/flex/class_proxy/templates/search.rb
 - lib/flex/configuration.rb
+- lib/flex/deprecation.rb
 - lib/flex/errors.rb
+- lib/flex/http_clients/base.rb
+- lib/flex/http_clients/loader.rb
 - lib/flex/http_clients/patron.rb
 - lib/flex/http_clients/rest_client.rb
-- lib/flex/instance_proxy/base.rb
-- lib/flex/instance_proxy/model.rb
-- lib/flex/instance_proxy/related_model.rb
-- lib/flex/loader.rb
 - lib/flex/logger.rb
-- lib/flex/manager.rb
-- lib/flex/model.rb
 - lib/flex/prog_bar.rb
 - lib/flex/rails.rb
-- lib/flex/rails/engine.rb
-- lib/flex/rails/helper.rb
-- lib/flex/related_model.rb
 - lib/flex/result.rb
 - lib/flex/result/bulk.rb
-- lib/flex/result/collection.rb
 - lib/flex/result/document.rb
-- lib/flex/result/indifferent_access.rb
+- lib/flex/result/multi_get.rb
 - lib/flex/result/search.rb
-- lib/flex/result/source_document.rb
-- lib/flex/result/source_search.rb
-- lib/flex/structure/indifferent_access.rb
-- lib/flex/structure/mergeable.rb
+- lib/flex/struct/array.rb
+- lib/flex/struct/hash.rb
+- lib/flex/struct/paginable.rb
+- lib/flex/struct/prunable.rb
+- lib/flex/struct/symbolize.rb
 - lib/flex/tasks.rb
 - lib/flex/template.rb
-- lib/flex/template/base.rb
-- lib/flex/template/info.rb
+- lib/flex/template/common.rb
+- lib/flex/template/logger.rb
 - lib/flex/template/partial.rb
 - lib/flex/template/search.rb
 - lib/flex/template/slim_search.rb
 - lib/flex/template/tags.rb
+- lib/flex/templates.rb
 - lib/flex/utility_methods.rb
 - lib/flex/utils.rb
 - lib/flex/variables.rb
-- lib/generators/flex/setup/setup_generator.rb
-- lib/generators/flex/setup/templates/flex_config.yml
-- lib/generators/flex/setup/templates/flex_dir/es.rb.erb
-- lib/generators/flex/setup/templates/flex_dir/es.yml.erb
-- lib/generators/flex/setup/templates/flex_dir/es_extender.rb.erb
-- lib/generators/flex/setup/templates/flex_initializer.rb.erb
-- lib/tasks/index.rake
-- test/flex.irt
-- test/flex/configuration.irt
-- test/irt_helper.rb
+- lib/tasks.rake
 homepage: http://github.com/ddnexus/flex
 licenses: []
 post_install_message: ! "________________________________________________________________________________\n\n
-  \                            FLEX INSTALLATION NOTES\n________________________________________________________________________________\n\nIn
-  order to use Flex, a supported http-client must be installed on this system.\n\nThe
-  suported http-client gems are \"patron\" and \"rest-client\".\n\nYou should install
-  \"patron\" (a libcurl based gem developed in C) for best\nperformances, or install
-  \"rest-client\" if you cannot use libcurl on your system.\n\nAs an alternative you
-  could eventually develop your own http-client interface\nand set the Flex::Configuration.http_client
-  option.\n\n________________________________________________________________________________\n"
+  \                            FLEX INSTALLATION NOTES\n________________________________________________________________________________\n\nNew
+  Documentation:  http://ddnexus.github.io/flex\n\nUpgrading Tutorial: http://ddnexus.github.io/flex/doc/7-Tutorials/2-Migrate-from-0.x.html\n\n________________________________________________________________________________\n"
 rdoc_options:
 - --charset=UTF-8
 require_paths:
@@ -177,9 +154,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.6
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
-summary: Ruby Client for ElasticSearch
+summary: The ultimate ruby client for elasticsearch
 test_files: []
-has_rdoc: 

data/bin/flexes

@@ -1,174 +0,0 @@
-#! /usr/bin/env ruby
-
-require 'optparse'
-require 'flex'
-
-options  = { }
-version  = File.read(File.expand_path('../../VERSION', __FILE__)).strip
-copy     = "flexes #{version} (c) 2012 Domizio Demichelis"
-optparse = OptionParser.new do |opts|
-
-  opts.banner = <<-banner
-
-    FlexEs:
-        Flex tool to dump/load data from/to ElasticSearch.
-    Usage:
-        flexes <command> [options]
-    <command>:
-        dump   dumps the data from one or more ElasticSearch indices
-        load   loads a dumpfile
-        stats  prints the full ElasticSearch stats
-
-    Notice: The load command will load the dump-file into ElasticSearch without removing any pre-existent data.
-            If you need fresh indices, use the flex:delete_indices and flex:create_indices rake tasks from your
-            application, which will also recreate the mapping.
-  banner
-
-
-  opts.separator ''
-  opts.separator 'Common options:'
-
-  options[:file] = './flexes.dump'
-  opts.on( '-f', '--file [FILE]', "The path of the dumpfile (default: '#{options[:file]}')" ) do |f|
-    options[:file] = f
-  end
-
-  opts.separator ''
-  opts.separator 'Dump options:'
-
-  options[:index] = nil
-  opts.on( '-i', '--index [INDEX_OR_INDICES]', Array, 'The index or comma separated indices to dump (default: all indices)' ) do |i|
-    options[:index] = i
-  end
-
-  options[:type] = nil
-  opts.on( '-t', '--type [TYPE_OR_TYPES]', Array, 'The type or comma separated types to dump (default: all types)' ) do |t|
-    options[:type] = t
-  end
-
-  options[:scroll] = '5m'
-  opts.on( '-s', '--scroll [TIME]', "The ElasticSearch scroll time (default: #{options[:scroll]})" ) do |s|
-    options[:scroll] = s
-  end
-
-  options[:size] = 50
-  opts.on( '-z', '--size [SIZE]', Integer, "The chunk size to dump per shard (default: #{options[:size]} * number of shards)" ) do |z|
-    options[:size] = z
-  end
-
-  opts.separator ''
-  opts.separator 'Load options:'
-
-  options[:timeout] = 20
-  opts.on( '-o', '--timeout [SECONDS]', Integer, "The http_client timeout for bulk loading (default: #{options[:timeout]} seconds)" ) do |o|
-    options[:timeout] = o
-  end
-
-  options[:batch_size] = 1000
-  opts.on( '-b', '--batch_size [BATCH_SIZE]', Integer, "The batch size to load (default: #{options[:batch_size]})" ) do |z|
-    options[:size] = z
-  end
-
-  opts.separator ''
-  opts.separator 'Other options:'
-
-  opts.on( '-v', '--version', 'Shows the version and exits' ) do
-    puts version
-    exit
-  end
-
-  opts.on_tail( '-h', '--help', 'Displays this screen' ) do
-    puts copy
-    puts opts
-    exit
-  end
-
-end
-
-optparse.parse!
-command = ARGV.first
-exec "#{$0} -h" if command.nil?
-puts copy
-
-case command
-
-when 'dump'
-  search_t    = Flex::Template::Search.new( {:query => {:match_all => {}}},
-                                            :params => { :search_type => 'scan',
-                                                         :scroll      => options[:scroll],
-                                                         :size        => options[:size],
-                                                         :fields      => '_source,*' } )
-  scroll_t    = Flex::Template.new( :get,
-                                    '/_search/scroll',
-                                    nil,
-                                    :params => { :scroll => options[:scroll] } )
-  search_res  = search_t.render options
-  scroll_id   = search_res['_scroll_id']
-  total_hits  = search_res['hits']['total']
-  total_count = 0
-  pbar        = Flex::ProgBar.new(total_hits)
-  dump_stats  = Hash.new { |hash, key| hash[key] = Hash.new{|h,k| h[k] = 0} }
-  file_size   = 0
-  File.open(options[:file], 'wb') do |file|
-    while (result = scroll_t.render(:data => scroll_id)) do
-      break if result['hits']['hits'] == []
-      lines = result['hits']['hits'].map do |h|
-                dump_stats[h['_index']][h['_type']] += 1
-                meta = { :_index => h['_index'],
-                         :_type  => h['_type'],
-                         :_id    => h['_id'] }
-                if h.has_key?('fields')
-                  h['fields'].each do |k,v|
-                    meta[k] = v if k[0] == '_'
-                  end
-                end
-                [ MultiJson.encode({'index' => meta}),
-                  MultiJson.encode(h['_source']) ].join("\n")
-              end
-      file.puts lines.join("\n")
-      scroll_id = result['_scroll_id']
-      total_count += lines.size
-      pbar.pbar.inc(lines.size)
-    end
-    file_size = file.size
-  end
-  formatted_file_size = file_size.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse
-  pbar.pbar.finish
-  puts "\n***** WARNING: Expected document to dump: #{total_hits}, dumped: #{total_count}. *****" \
-       unless total_hits == total_count
-  puts "\nDumped #{total_count} documents to #{options[:file]} (size: #{formatted_file_size} bytes)"
-  puts dump_stats.to_yaml
-
-when 'load'
-  Flex::Configuration.http_client_options[:timeout] = options[:timeout]
-  chunk_size = options[:batch_size] * 2    # 2 lines per doc
-  lines      = ''
-  line_count = 0
-  file       = File.open(options[:file])
-  file.lines{ line_count += 1 }
-  file.rewind
-  pbar = Flex::ProgBar.new(line_count / 2, options[:batch_size])
-  file.lines do |line|
-    lines << line
-    if file.lineno % chunk_size == 0
-      result = Flex.bulk :lines => lines
-      lines  = ''
-      pbar.process_result(result, options[:batch_size])
-    end
-  end
-  # last chunk
-  unless lines == ''
-    result = Flex.bulk :lines => lines
-    pbar.process_result(result, (file.lineno % chunk_size) / 2)
-  end
-  file.close
-  pbar.finish
-
-when 'stats'
-  puts '>> puts Flex.stats.to_yaml'
-  puts Flex.stats.to_yaml
-
-else
-  puts "unknown command: #{command.inspect}"
-
-end