flex 0.4.2 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. data/LICENSE +1 -1
  2. data/README.md +46 -7
  3. data/VERSION +1 -1
  4. data/flex.gemspec +13 -19
  5. data/lib/flex.rb +66 -392
  6. data/lib/flex/api_stubs.rb +1268 -0
  7. data/lib/flex/api_templates/cluster_api.yml +94 -0
  8. data/lib/flex/api_templates/core_api.yml +202 -0
  9. data/lib/flex/api_templates/indices_api.yml +304 -0
  10. data/lib/flex/class_proxy/base.rb +20 -6
  11. data/lib/flex/class_proxy/templates.rb +97 -0
  12. data/lib/flex/class_proxy/templates/doc.rb +91 -0
  13. data/lib/flex/class_proxy/templates/search.rb +72 -0
  14. data/lib/flex/configuration.rb +17 -49
  15. data/lib/flex/deprecation.rb +153 -0
  16. data/lib/flex/errors.rb +2 -1
  17. data/lib/flex/http_clients/base.rb +15 -0
  18. data/lib/flex/http_clients/loader.rb +51 -0
  19. data/lib/flex/http_clients/patron.rb +9 -7
  20. data/lib/flex/http_clients/rest_client.rb +6 -8
  21. data/lib/flex/logger.rb +24 -3
  22. data/lib/flex/prog_bar.rb +11 -6
  23. data/lib/flex/rails.rb +1 -13
  24. data/lib/flex/result.rb +8 -2
  25. data/lib/flex/result/document.rb +42 -13
  26. data/lib/flex/result/multi_get.rb +24 -0
  27. data/lib/flex/result/search.rb +1 -24
  28. data/lib/flex/struct/array.rb +25 -0
  29. data/lib/flex/struct/hash.rb +105 -0
  30. data/lib/flex/{result/collection.rb → struct/paginable.rb} +16 -9
  31. data/lib/flex/struct/prunable.rb +60 -0
  32. data/lib/flex/struct/symbolize.rb +27 -0
  33. data/lib/flex/tasks.rb +26 -86
  34. data/lib/flex/template.rb +60 -120
  35. data/lib/flex/template/common.rb +42 -0
  36. data/lib/flex/template/logger.rb +66 -0
  37. data/lib/flex/template/partial.rb +12 -15
  38. data/lib/flex/template/search.rb +6 -6
  39. data/lib/flex/template/slim_search.rb +1 -1
  40. data/lib/flex/template/tags.rb +19 -9
  41. data/lib/flex/templates.rb +20 -0
  42. data/lib/flex/utility_methods.rb +80 -89
  43. data/lib/flex/utils.rb +68 -25
  44. data/lib/flex/variables.rb +55 -4
  45. data/lib/tasks.rake +28 -0
  46. metadata +61 -85
  47. data/bin/flexes +0 -174
  48. data/lib/flex/api_methods.yml +0 -108
  49. data/lib/flex/class_proxy/loader.rb +0 -102
  50. data/lib/flex/class_proxy/model.rb +0 -45
  51. data/lib/flex/class_proxy/model_sync.rb +0 -23
  52. data/lib/flex/class_proxy/related_model.rb +0 -23
  53. data/lib/flex/instance_proxy/base.rb +0 -29
  54. data/lib/flex/instance_proxy/model.rb +0 -102
  55. data/lib/flex/instance_proxy/related_model.rb +0 -7
  56. data/lib/flex/loader.rb +0 -18
  57. data/lib/flex/manager.rb +0 -61
  58. data/lib/flex/model.rb +0 -24
  59. data/lib/flex/rails/engine.rb +0 -23
  60. data/lib/flex/rails/helper.rb +0 -16
  61. data/lib/flex/related_model.rb +0 -16
  62. data/lib/flex/result/indifferent_access.rb +0 -11
  63. data/lib/flex/result/source_document.rb +0 -63
  64. data/lib/flex/result/source_search.rb +0 -32
  65. data/lib/flex/structure/indifferent_access.rb +0 -44
  66. data/lib/flex/structure/mergeable.rb +0 -21
  67. data/lib/flex/template/base.rb +0 -41
  68. data/lib/flex/template/info.rb +0 -68
  69. data/lib/generators/flex/setup/setup_generator.rb +0 -48
  70. data/lib/generators/flex/setup/templates/flex_config.yml +0 -16
  71. data/lib/generators/flex/setup/templates/flex_dir/es.rb.erb +0 -18
  72. data/lib/generators/flex/setup/templates/flex_dir/es.yml.erb +0 -19
  73. data/lib/generators/flex/setup/templates/flex_dir/es_extender.rb.erb +0 -17
  74. data/lib/generators/flex/setup/templates/flex_initializer.rb.erb +0 -44
  75. data/lib/tasks/index.rake +0 -17
  76. data/test/flex.irt +0 -143
  77. data/test/flex/configuration.irt +0 -53
  78. data/test/irt_helper.rb +0 -12
@@ -8,6 +8,7 @@ module Flex
8
8
  class ExistingIndexError < StandardError; end
9
9
  class MissingHttpClientError < StandardError; end
10
10
  class MissingParentError < StandardError; end
11
+ class MissingVariableError < StandardError; end
11
12
 
12
13
  class HttpError < StandardError
13
14
 
@@ -27,7 +28,7 @@ module Flex
27
28
  end
28
29
 
29
30
  def to_s
30
- log = "#{@caller_line}\n" if @caller_line
31
+ log = "#@caller_line\n" if @caller_line
31
32
  "#{log}#{status}: #{body}"
32
33
  end
33
34
 
@@ -0,0 +1,15 @@
1
+ module Flex
2
+ module HttpClients
3
+ class Base
4
+
5
+ attr_accessor :options, :base_uri, :raise_proc
6
+
7
+ def initialize(base_uri='http://localhost:9200', options={})
8
+ @options = options
9
+ @base_uri = base_uri
10
+ @raise_proc = proc{|response| response.status >= 400}
11
+ end
12
+
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,51 @@
1
+ module Flex
2
+ module HttpClients
3
+
4
+ class Dummy
5
+ def request(*)
6
+ raise MissingHttpClientError,
7
+ 'you should install the gem "patron" (recommended for performances) or "rest-client", ' +
8
+ 'or provide your own http-client interface and set Flex::Configuration.http_client'
9
+ end
10
+ end
11
+
12
+ module Loader
13
+
14
+ extend self
15
+
16
+ def new_http_client
17
+ if Gem::Specification.respond_to?(:find_all_by_name)
18
+ case
19
+ # terrible way to check whether a gem is available.
20
+ # Gem.available? was just perfect: that's probably the reason it has been deprecated!
21
+ # https://github.com/rubygems/rubygems/issues/176
22
+ when Gem::Specification::find_all_by_name('patron').any? then require_patron
23
+ when Gem::Specification::find_all_by_name('rest-client').any? then require_rest_client
24
+ else Dummy.new
25
+ end
26
+ else
27
+ case
28
+ when Gem.available?('patron') then require_patron
29
+ when Gem.available?('rest-client') then require_rest_client
30
+ else Dummy.new
31
+ end
32
+ end
33
+ end
34
+
35
+ private
36
+
37
+ def require_patron
38
+ require 'patron'
39
+ require 'flex/http_clients/patron'
40
+ Patron.new
41
+ end
42
+
43
+ def require_rest_client
44
+ require 'rest-client'
45
+ require 'flex/http_clients/rest_client'
46
+ RestClient.new
47
+ end
48
+
49
+ end
50
+ end
51
+ end
@@ -1,14 +1,16 @@
1
1
  module Flex
2
2
  module HttpClients
3
- module Patron
4
- extend self
3
+ class Patron < Base
5
4
 
6
5
  def request(method, path, data=nil)
7
- options = Configuration.http_client_options
8
- options = options.merge(:data => data) if data
9
- session.request method.to_s.downcase.to_sym, path, {}, options
6
+ # patron would raise an error for :post and :put requests with no data
7
+ # and elasticsearch ignores the data when it expects no data,
8
+ # so we silence patron by adding some dummy data
9
+ data = {} if (method == 'POST' || method == 'PUT') && data.nil?
10
+ opts = options.merge(:data => data)
11
+ session.request method.to_s.downcase.to_sym, path, {}, opts
10
12
  rescue ::Patron::TimeoutError
11
- session.request method.to_s.downcase.to_sym, path, {}, options
13
+ session.request method.to_s.downcase.to_sym, path, {}, opts
12
14
  end
13
15
 
14
16
  private
@@ -17,7 +19,7 @@ module Flex
17
19
  Thread.current[:flex_patron_session] ||= begin
18
20
  sess = ::Patron::Session.new
19
21
  sess.headers['User-Agent'] = "flex-#{Flex::VERSION}"
20
- sess.base_url = Configuration.base_uri
22
+ sess.base_url = base_uri
21
23
  sess
22
24
  end
23
25
  end
@@ -1,15 +1,13 @@
1
1
  module Flex
2
2
  module HttpClients
3
- module RestClient
4
- extend self
3
+ class RestClient < Base
5
4
 
6
5
  def request(method, path, data=nil)
7
- options = Configuration.http_client_options
8
- url = "#{Configuration.base_uri}#{path}"
9
- args = options.merge( :method => method.to_s.downcase.to_sym,
10
- :url => url,
11
- :payload => data )
12
- response = ::RestClient::Request.new( args ).execute
6
+ url = "#{base_uri}#{path}"
7
+ opts = options.merge( :method => method.to_s.downcase.to_sym,
8
+ :url => url,
9
+ :payload => data )
10
+ response = ::RestClient::Request.new( opts ).execute
13
11
  extend_response(response, url)
14
12
 
15
13
  rescue ::RestClient::ExceptionWithResponse => e
@@ -3,13 +3,34 @@ require 'logger'
3
3
  module Flex
4
4
  class Logger < ::Logger
5
5
 
6
+
7
+ attr_accessor :debug_variables, :debug_request, :debug_result, :curl_format
8
+
6
9
  def initialize(*)
7
10
  super
8
- self.level = ::Logger::INFO
9
- self.progname = "FLEX"
11
+ self.level = ::Logger::WARN
12
+ self.progname = 'FLEX'
10
13
  self.formatter = proc do |severity, datetime, progname, msg|
11
- "#{msg}\n"
14
+ flex_format(severity, msg)
12
15
  end
16
+ @debug_variables = true
17
+ @debug_request = true
18
+ @debug_result = true
19
+ @curl_format = false
20
+ end
21
+
22
+ def flex_format(severity, msg)
23
+ prefix = Dye.dye(" FLEX-#{severity} ", "FLEX-#{severity}:", :blue, :bold, :reversed) + ' '
24
+ msg.split("\n").map{|l| prefix + l}.join("\n") + "\n"
25
+ end
26
+
27
+ # force color in console (used with jruby)
28
+ def color=(bool)
29
+ Dye.color = bool
30
+ end
31
+
32
+ def color
33
+ Dye.color?
13
34
  end
14
35
 
15
36
  end
@@ -1,7 +1,7 @@
1
1
  module Flex
2
2
  class ProgBar
3
3
 
4
- attr_reader :pbar
4
+ attr_reader :pbar, :total_count
5
5
 
6
6
  def initialize(total_count, batch_size=nil, prefix_message=nil)
7
7
  @total_count = total_count
@@ -18,16 +18,21 @@ module Flex
18
18
  end
19
19
 
20
20
  def process_result(result, inc)
21
- Configuration.logger.error "[FLEX] Failed load:\n#{result.failed.to_yaml}" unless result.failed.size == 0
22
- @failed_count += result.failed.size
23
- @successful_count += result.successful.size
21
+ unless result.nil? || result.empty?
22
+ unless result.failed.size == 0
23
+ Conf.logger.error "Failed load:\n#{result.failed.to_yaml}"
24
+ @pbar.bar_mark = 'F'
25
+ end
26
+ @failed_count += result.failed.size
27
+ @successful_count += result.successful.size
28
+ end
24
29
  @pbar.inc(inc)
25
30
  end
26
31
 
27
32
  def finish
28
33
  @pbar.finish
29
- puts "Processed #{@total_count}. Successful #{@successful_count}. Skipped #{@total_count - @successful_count - @failed_count}. Failed #{@failed_count}."
30
- puts "See the log for the details about the failed import." unless @failed_count == 0
34
+ puts "Processed #@total_count. Successful #@successful_count. Skipped #{@total_count - @successful_count - @failed_count}. Failed #@failed_count."
35
+ puts 'See the log for the details about the failures.' unless @failed_count == 0
31
36
  end
32
37
 
33
38
  end
@@ -1,13 +1 @@
1
- require 'flex'
2
- require 'rails'
3
- require 'flex/rails/helper'
4
-
5
- if ::Rails.respond_to?(:version) && ::Rails.version.to_i >= 3
6
- require 'flex/rails/engine'
7
- else
8
- Flex::Configuration.configure do |c|
9
- c.config_file = ::Rails.root.join('config', 'flex.yml').to_s
10
- c.flex_dir = ::Rails.root.join('app', 'flex').to_s
11
- c.debug = ::Rails.env.development?
12
- end
13
- end
1
+ raise NotImplementedError, %(The "flex/rails" file has been replaced by the "flex-rails" gem. Please, use "flex-rails" in place of "flex" in Rails applications (change "gem 'flex', require => 'flex/rails'" to "gem 'flex-rails'" in your Gemfile))
@@ -1,18 +1,24 @@
1
1
  module Flex
2
2
  class Result < ::Hash
3
3
 
4
- attr_reader :template, :variables, :response
4
+ attr_reader :template, :response
5
+ attr_accessor :variables
5
6
 
6
7
  def initialize(template, variables, response, result=nil)
7
8
  @template = template
8
9
  @variables = variables
9
10
  @response = response
10
11
  replace result || !response.body.empty? && MultiJson.decode(response.body) || return
11
- Configuration.result_extenders.each do |ext|
12
+ Conf.result_extenders.each do |ext|
12
13
  next if ext.respond_to?(:should_extend?) && !ext.should_extend?(self)
13
14
  extend ext
14
15
  end
15
16
  end
16
17
 
18
+ def to_flex_result(force=false)
19
+ return self if variables[:context].nil? || variables[:raw_result] &&! force
20
+ variables[:context].respond_to?(:flex_result) ? variables[:context].flex_result(self) : self
21
+ end
22
+
17
23
  end
18
24
  end
@@ -1,36 +1,65 @@
1
1
  module Flex
2
2
  class Result
3
3
 
4
- # adds sugar to documents with the following structure:
4
+ # adds sugar to documents with the following structure (_source is optional):
5
5
  #
6
6
  # {
7
7
  # "_index" : "twitter",
8
8
  # "_type" : "tweet",
9
9
  # "_id" : "1",
10
+ # "_source" : {
11
+ # "user" : "kimchy",
12
+ # "postDate" : "2009-11-15T14:12:12",
13
+ # "message" : "trying out Elastic Search"
14
+ # }
10
15
  # }
11
16
 
12
17
  module Document
13
18
 
14
- META = %w[_index _type _id]
15
-
16
19
  # extend if result has a structure like a document
17
20
  def self.should_extend?(obj)
18
- META.all? {|k| obj.has_key?(k)}
21
+ %w[_index _type _id].all? {|k| obj.has_key?(k)}
22
+ end
23
+
24
+ def respond_to?(meth, private=false)
25
+ smeth = meth.to_s
26
+ readers.has_key?(smeth) || has_key?(smeth) || has_key?("_#{smeth}") || super
19
27
  end
20
28
 
21
- META.each do |m|
22
- define_method(m){self["#{m}"]}
29
+ # exposes _source and readers: automatically supply object-like reader access
30
+ # also expose meta readers like _id, _source, etc, also callable without the leading '_'
31
+ def method_missing(meth, *args, &block)
32
+ smeth = meth.to_s
33
+ case
34
+ # field name
35
+ when readers.has_key?(smeth)
36
+ readers[smeth]
37
+ # result item
38
+ when has_key?(smeth)
39
+ self[smeth]
40
+ # result item called without the '_' prefix
41
+ when has_key?("_#{smeth}")
42
+ self["_#{smeth}"]
43
+ else
44
+ super
45
+ end
23
46
  end
24
47
 
25
- def mapped_class(should_raise=false)
26
- @mapped_class ||= Manager.type_class_map["#{_index}/#{_type}"]
27
- rescue NameError
28
- raise DocumentMappingError, "the '#{_index}/#{_type}' document cannot be mapped to any class." \
29
- if should_raise
48
+ # used to get the unprefixed (by live-reindex) index name
49
+ def index_basename
50
+ @index_basename ||= self['_index'].sub(/^\d{14}_/, '')
30
51
  end
31
52
 
32
- def load
33
- mapped_class.find _id
53
+
54
+ private
55
+
56
+ def readers
57
+ @readers ||= begin
58
+ readers = (self['_source']||{}).merge(self['fields']||{})
59
+ # flattened reader for multi_readers or attachment readers
60
+ readers.keys.each{|k| readers[k.gsub('.','_')] = readers.delete(k) if k.include?('.')}
61
+ readers
62
+ end
34
63
  end
35
64
 
36
65
  end
@@ -0,0 +1,24 @@
1
+ module Flex
2
+ class Result
3
+ module MultiGet
4
+
5
+ # extend if result comes from a search url
6
+ def self.should_extend?(result)
7
+ result.response.url =~ /\b_mget\b/ && result['docs']
8
+ end
9
+
10
+ # extend the hits results on extended
11
+ def self.extended(result)
12
+ result['docs'].each { |h| h.extend(Document) }
13
+ result['docs'].extend Struct::Paginable
14
+ result['docs'].setup(result['docs'].size, result.variables)
15
+ end
16
+
17
+ def docs
18
+ self['docs']
19
+ end
20
+ alias_method :collection, :docs
21
+
22
+ end
23
+ end
24
+ end
@@ -10,7 +10,7 @@ module Flex
10
10
  # extend the hits results on extended
11
11
  def self.extended(result)
12
12
  result['hits']['hits'].each { |h| h.extend(Document) }
13
- result['hits']['hits'].extend Collection
13
+ result['hits']['hits'].extend Struct::Paginable
14
14
  result['hits']['hits'].setup(result['hits']['total'], result.variables)
15
15
  end
16
16
 
@@ -23,29 +23,6 @@ module Flex
23
23
  self['facets']
24
24
  end
25
25
 
26
- def loaded_collection
27
- @loaded_collection ||= begin
28
- records = []
29
- # returns a structure like {Comment=>[{"_id"=>"123", ...}, {...}], BlogPost=>[...]}
30
- h = Utils.group_array_by(collection) do |d|
31
- d.mapped_class(should_raise=true)
32
- end
33
- h.each do |klass, docs|
34
- records |= klass.find(docs.map(&:_id))
35
- end
36
- class_ids = collection.map { |d| [d.mapped_class.to_s, d._id] }
37
- # Reorder records to preserve order from search results
38
- records = class_ids.map do |class_str, id|
39
- records.detect do |record|
40
- record.class.to_s == class_str && record.id.to_s == id.to_s
41
- end
42
- end
43
- records.extend Collection
44
- records.setup(self['hits']['total'], variables)
45
- records
46
- end
47
- end
48
-
49
26
  end
50
27
  end
51
28
  end
@@ -0,0 +1,25 @@
1
+ module Flex
2
+ module Struct
3
+ class Array < ::Array
4
+
5
+ include Symbolize
6
+
7
+ def push(*vals)
8
+ super *symbolize(vals)
9
+ end
10
+
11
+ def <<(val)
12
+ super symbolize(val)
13
+ end
14
+
15
+ def insert(*vals)
16
+ super *symbolize(vals)
17
+ end
18
+
19
+ def unshift(*vals)
20
+ super *symbolize(vals)
21
+ end
22
+
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,105 @@
1
+ module Flex
2
+ module Struct
3
+ class Hash < ::Hash
4
+ include Symbolize
5
+
6
+ def initialize
7
+ super do |hash, key|
8
+ if key[-1] == '!'
9
+ klass = (key[0] == '_' ? Array : Hash)
10
+ hash[clean_key(key)] = klass.new
11
+ end
12
+ end
13
+ end
14
+
15
+ def merge(hash)
16
+ super symbolize(hash)
17
+ end
18
+
19
+ def merge!(hash)
20
+ super symbolize(hash)
21
+ end
22
+
23
+ def store(key, val)
24
+ if key[-1] == '='
25
+ super key[0..-2].to_sym, val.extend(AsIs)
26
+ else
27
+ super clean_key(key), symbolize(val)
28
+ end
29
+ end
30
+ alias_method :[]=, :store
31
+
32
+ def fetch(key, *rest, &block)
33
+ cleaned = clean_key(key)
34
+ super has_key?(cleaned) ? cleaned : key.to_sym, *rest, &block
35
+ end
36
+
37
+ def [](key)
38
+ cleaned = clean_key(key)
39
+ super has_key?(cleaned) ? cleaned : key.to_sym
40
+ end
41
+
42
+ def deep_merge(*hashes)
43
+ dupe = deep_dup(self)
44
+ hashes.each {|h2| dupe.replace(deep_merge_hash(dupe,h2))}
45
+ dupe
46
+ end
47
+
48
+ def deep_merge!(*hashes)
49
+ replace deep_merge(*hashes)
50
+ end
51
+
52
+ module Nil
53
+ def method_missing(*)
54
+ self
55
+ end
56
+ end
57
+
58
+ def try(key)
59
+ has_key?(key) ? self[key] : nil.extend(Nil)
60
+ end
61
+
62
+ def try_delete(key, *rest, &block)
63
+ val = delete clean_key(key), *rest, &block
64
+ val.nil? ? nil.extend(Nil) : val
65
+ end
66
+
67
+
68
+ private
69
+
70
+ def clean_key(key)
71
+ key[-1] == '!' ? key[0..-2].to_sym : key.to_sym
72
+ end
73
+
74
+ def deep_merge_hash(h1, h2)
75
+ h2 ||= {}
76
+ h1.merge(h2) do |key, oldval, newval|
77
+ case
78
+ when oldval.is_a?(Hash) && newval.is_a?(Hash)
79
+ deep_merge_hash(oldval, newval)
80
+ when oldval.is_a?(Array) && newval.is_a?(Array)
81
+ oldval | newval
82
+ else
83
+ newval
84
+ end
85
+ end
86
+ end
87
+
88
+ def deep_dup(obj)
89
+ case obj
90
+ when ::Hash, Flex::Struct::Hash
91
+ h = obj.dup
92
+ h.each_pair do |k,v|
93
+ h[k] = deep_dup(v)
94
+ end
95
+ h
96
+ when ::Array, Flex::Struct::Array
97
+ obj.map{|i| deep_dup(i)}
98
+ else
99
+ obj
100
+ end
101
+ end
102
+
103
+ end
104
+ end
105
+ end