flex 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/LICENSE +20 -0
  2. data/README.md +20 -0
  3. data/VERSION +1 -0
  4. data/flex.gemspec +43 -0
  5. data/lib/flex.rb +418 -0
  6. data/lib/flex/api_methods.yml +108 -0
  7. data/lib/flex/class_proxy.rb +12 -0
  8. data/lib/flex/configuration.rb +57 -0
  9. data/lib/flex/errors.rb +42 -0
  10. data/lib/flex/http_clients/patron.rb +27 -0
  11. data/lib/flex/http_clients/rest_client.rb +38 -0
  12. data/lib/flex/loader.rb +116 -0
  13. data/lib/flex/logger.rb +16 -0
  14. data/lib/flex/model.rb +24 -0
  15. data/lib/flex/model/class_proxy.rb +45 -0
  16. data/lib/flex/model/instance_proxy.rb +101 -0
  17. data/lib/flex/model/manager.rb +67 -0
  18. data/lib/flex/rails.rb +12 -0
  19. data/lib/flex/rails/engine.rb +23 -0
  20. data/lib/flex/rails/helper.rb +16 -0
  21. data/lib/flex/related_model.rb +16 -0
  22. data/lib/flex/related_model/class_proxy.rb +23 -0
  23. data/lib/flex/related_model/class_sync.rb +23 -0
  24. data/lib/flex/related_model/instance_proxy.rb +28 -0
  25. data/lib/flex/result.rb +18 -0
  26. data/lib/flex/result/bulk.rb +20 -0
  27. data/lib/flex/result/collection.rb +51 -0
  28. data/lib/flex/result/document.rb +38 -0
  29. data/lib/flex/result/indifferent_access.rb +11 -0
  30. data/lib/flex/result/search.rb +51 -0
  31. data/lib/flex/result/source_document.rb +63 -0
  32. data/lib/flex/result/source_search.rb +32 -0
  33. data/lib/flex/structure/indifferent_access.rb +44 -0
  34. data/lib/flex/structure/mergeable.rb +21 -0
  35. data/lib/flex/tasks.rb +141 -0
  36. data/lib/flex/template.rb +187 -0
  37. data/lib/flex/template/base.rb +29 -0
  38. data/lib/flex/template/info.rb +50 -0
  39. data/lib/flex/template/partial.rb +31 -0
  40. data/lib/flex/template/search.rb +30 -0
  41. data/lib/flex/template/slim_search.rb +13 -0
  42. data/lib/flex/template/tags.rb +46 -0
  43. data/lib/flex/utility_methods.rb +140 -0
  44. data/lib/flex/utils.rb +59 -0
  45. data/lib/flex/variables.rb +11 -0
  46. data/lib/generators/flex/setup/setup_generator.rb +51 -0
  47. data/lib/generators/flex/setup/templates/flex_config.yml +16 -0
  48. data/lib/generators/flex/setup/templates/flex_dir/es.rb.erb +18 -0
  49. data/lib/generators/flex/setup/templates/flex_dir/es.yml.erb +19 -0
  50. data/lib/generators/flex/setup/templates/flex_dir/es_extender.rb.erb +17 -0
  51. data/lib/generators/flex/setup/templates/flex_initializer.rb.erb +44 -0
  52. data/lib/tasks/index.rake +23 -0
  53. data/test/flex.irt +143 -0
  54. data/test/flex/configuration.irt +53 -0
  55. data/test/irt_helper.rb +12 -0
  56. metadata +211 -0
@@ -0,0 +1,38 @@
1
+ module Flex
2
+ class Result
3
+
4
+ # adds sugar to documents with the following structure:
5
+ #
6
+ # {
7
+ # "_index" : "twitter",
8
+ # "_type" : "tweet",
9
+ # "_id" : "1",
10
+ # }
11
+
12
+ module Document
13
+
14
+ META = %w[_index _type _id]
15
+
16
+ # extend if result has a structure like a document
17
+ def self.should_extend?(obj)
18
+ META.all? {|k| obj.has_key?(k)}
19
+ end
20
+
21
+ META.each do |m|
22
+ define_method(m){self["#{m}"]}
23
+ end
24
+
25
+ def mapped_class(should_raise=false)
26
+ @mapped_class ||= Model::Manager.type_class_map["#{_index}/#{_type}"]
27
+ rescue NameError
28
+ raise DocumentMappingError, "the '#{_index}/#{_type}' document cannot be mapped to any class." \
29
+ if should_raise
30
+ end
31
+
32
+ def load
33
+ mapped_class.find _id
34
+ end
35
+
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,11 @@
1
+ module Flex
2
+ module Result
3
+ module IndifferentAccess
4
+
5
+ def self.extended(result)
6
+ result.extend Structure::IndifferentAccess
7
+ end
8
+
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,51 @@
1
+ module Flex
2
+ class Result
3
+ module Search
4
+
5
+ # extend if result comes from a search url
6
+ def self.should_extend?(result)
7
+ result.response.url =~ /\b_m?search\b/ && result['hits']
8
+ end
9
+
10
+ # extend the hits results on extended
11
+ def self.extended(result)
12
+ result['hits']['hits'].each { |h| h.extend(Document) }
13
+ result['hits']['hits'].extend Collection
14
+ result['hits']['hits'].setup(result['hits']['total'], result.variables)
15
+ end
16
+
17
+ def collection
18
+ self['hits']['hits']
19
+ end
20
+ alias_method :documents, :collection
21
+
22
+ def facets
23
+ self['facets']
24
+ end
25
+
26
+ def loaded_collection
27
+ @loaded_collection ||= begin
28
+ records = []
29
+ # returns a structure like {Comment=>[{"_id"=>"123", ...}, {...}], BlogPost=>[...]}
30
+ h = Utils.group_array_by(collection) do |d|
31
+ d.mapped_class(should_raise=true)
32
+ end
33
+ h.each do |klass, docs|
34
+ records |= klass.find(docs.map(&:_id))
35
+ end
36
+ class_ids = collection.map { |d| [d.mapped_class.to_s, d._id] }
37
+ # Reorder records to preserve order from search results
38
+ records = class_ids.map do |class_str, id|
39
+ records.detect do |record|
40
+ record.class.to_s == class_str && record.id.to_s == id.to_s
41
+ end
42
+ end
43
+ records.extend Collection
44
+ records.setup(self['hits']['total'], variables)
45
+ records
46
+ end
47
+ end
48
+
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,63 @@
1
+ module Flex
2
+ class Result
3
+
4
+ # adds sugar to documents with the following structure:
5
+ #
6
+ # {
7
+ # "_index" : "twitter",
8
+ # "_type" : "tweet",
9
+ # "_id" : "1",
10
+ # "_source" : {
11
+ # "user" : "kimchy",
12
+ # "postDate" : "2009-11-15T14:12:12",
13
+ # "message" : "trying out Elastic Search"
14
+ # }
15
+ # }
16
+
17
+ module SourceDocument
18
+
19
+ # extend if result has a structure like a document
20
+ def self.should_extend?(obj)
21
+ %w[_index _type _id _source].all? {|k| obj.has_key?(k)}
22
+ end
23
+
24
+ # exposes _source: automatically supply object-like reader access
25
+ # also expose meta fields like _id, _source, etc, also for methods without the leading '_'
26
+ def method_missing(meth, *args, &block)
27
+ case
28
+ when meth.to_s =~ /^_/ && has_key?(meth.to_s) then self[meth.to_s]
29
+ when self['_source'].has_key?(meth.to_s) then self['_source'][meth.to_s]
30
+ when has_key?("_#{meth.to_s}") then self["_#{meth.to_s}"]
31
+ else super
32
+ end
33
+ end
34
+
35
+ # returns the _source hash with an added id (if missing))
36
+ def to_attributes
37
+ {'id' => _id}.merge(_source)
38
+ end
39
+
40
+ # creates an instance of a mapped or computed class, falling back to OpenStruct
41
+ def to_mapped
42
+ to(mapped_class || OpenStruct)
43
+ end
44
+
45
+ # experimental: creates an instance of klass out of to_attributes
46
+ # we should probably reset the id to the original document _id
47
+ # but be sure the record is read-only
48
+ def to(klass)
49
+ obj = klass.new(to_attributes)
50
+ case
51
+ when defined?(ActiveRecord::Base) && obj.is_a?(ActiveRecord::Base)
52
+ obj.readonly!
53
+ when defined?(Mongoid::Document) && obj.is_a?(Mongoid::Document)
54
+ # TODO: make it readonly
55
+ when obj.is_a?(OpenStruct)
56
+ # TODO: anythig to extend?
57
+ end
58
+ obj
59
+ end
60
+
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,32 @@
1
+ module Flex
2
+ class Result
3
+ module SourceSearch
4
+
5
+ # extend if result comes from a search url and does not contain an empty fields param (no _source))
6
+ def self.should_extend?(result)
7
+ result.response.url =~ /\b_m?search\b/ &&
8
+ !result['hits']['hits'].empty? && result['hits']['hits'].first.has_key?('_source')
9
+ end
10
+
11
+ # extend the hits results on extended
12
+ def self.extended(result)
13
+ result['hits']['hits'].each { |h| h.extend(SourceDocument) }
14
+ end
15
+
16
+ # experimental
17
+ # returns an array of document mapped objects
18
+ def mapped_collection
19
+ @mapped_collection ||= begin
20
+ docs = self['hits']['hits'].map do |h|
21
+ raise NameError, "no '_source' found in hit #{h.inspect} " \
22
+ unless h.respond_to(:to_mapped)
23
+ h.to_mapped
24
+ end
25
+ docs.extend Collection
26
+ docs.setup(self['hits']['total'], variables)
27
+ end
28
+ end
29
+
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,44 @@
1
+ module Flex
2
+ module Structure
3
+ # allows to use both Symbol or String keys to access the same values in a Hash
4
+ module IndifferentAccess
5
+
6
+ def [](k)
7
+ get_value(k)
8
+ end
9
+
10
+ def []=(k,v)
11
+ # default to_s for storing new keys
12
+ has_key?(k) ? super : super(k.to_s, v)
13
+ end
14
+
15
+ def to_hash
16
+ self
17
+ end
18
+
19
+ private
20
+
21
+ def get_value(k)
22
+ val = fetch_val(k)
23
+ case val
24
+ when Hash
25
+ val.extend IndifferentAccess
26
+ when Array
27
+ val.each {|v| v.extend IndifferentAccess if v.is_a?(Hash)}
28
+ end
29
+ val
30
+ end
31
+
32
+ def fetch_val(k)
33
+ v = fetch(k, nil)
34
+ return v unless v.nil?
35
+ if k.is_a?(String)
36
+ v = fetch(k.to_sym, nil)
37
+ return v unless v.nil?
38
+ end
39
+ fetch(k.to_s, nil) if k.is_a?(Symbol)
40
+ end
41
+
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,21 @@
1
+ module Flex
2
+ module Structure
3
+ # allows deep merge between Hashes
4
+ module Mergeable
5
+
6
+ def deep_merge(*hashes)
7
+ Utils.deep_merge_hashes(self, *hashes)
8
+ end
9
+
10
+ def deep_merge!(*hashes)
11
+ replace deep_merge(*hashes)
12
+ end
13
+ alias_method :add, :deep_merge!
14
+
15
+ def deep_dup
16
+ Marshal.load(Marshal.dump(self))
17
+ end
18
+
19
+ end
20
+ end
21
+ end
data/lib/flex/tasks.rb ADDED
@@ -0,0 +1,141 @@
1
+ module Flex
2
+ module Tasks
3
+ extend self
4
+
5
+ def create_indices
6
+ indices.each do |index|
7
+ delete_index(index) if ENV['FORCE']
8
+ raise ExistingIndexError, "#{index.inspect} already exists. Please use FORCE=1 if you want to delete it first." \
9
+ if exist?(index)
10
+ create(index)
11
+ end
12
+ end
13
+
14
+ def delete_indices
15
+ indices.each { |index| delete_index(index) }
16
+ end
17
+
18
+ def import_models
19
+ require 'progressbar'
20
+
21
+ Configuration.http_client_options[:timeout] = ENV['TIMEOUT'].to_i if ENV['TIMEOUT']
22
+ Configuration.http_client_options[:timeout] ||= 20
23
+ Configuration.debug = !!ENV['FLEX_DEBUG']
24
+ batch_size = ENV['BATCH_SIZE'] && ENV['BATCH_SIZE'].to_i || 1000
25
+ options = {}
26
+ if ENV['IMPORT_OPTIONS']
27
+ ENV['IMPORT_OPTIONS'].split('&').each do |pair|
28
+ k, v = pair.split('=')
29
+ options[k.to_sym] = v
30
+ end
31
+ end
32
+ deleted = []
33
+
34
+ models.each do |klass|
35
+ index = klass.flex.index
36
+
37
+ if ENV['FORCE']
38
+ unless deleted.include?(index)
39
+ delete_index(index)
40
+ deleted << index
41
+ puts "#{index} index deleted"
42
+ end
43
+ end
44
+
45
+ unless exist?(index)
46
+ create(index)
47
+ puts "#{index} index created"
48
+ end
49
+
50
+ if defined?(Mongoid::Document) && klass.ancestors.include?(Mongoid::Document)
51
+ def klass.find_in_batches(options={})
52
+ 0.step(count, options[:batch_size]) do |offset|
53
+ yield limit(options[:batch_size]).skip(offset).to_a
54
+ end
55
+ end
56
+ end
57
+
58
+ unless klass.respond_to?(:find_in_batches)
59
+ STDERR.puts "[ERROR] Class #{klass} does not respond to :find_in_batches. Skipped."
60
+ next
61
+ end
62
+
63
+ total_count = klass.count
64
+ successful_count = 0
65
+ failed = []
66
+
67
+ pbar = ProgressBar.new('processing...', total_count)
68
+ pbar.clear
69
+ pbar.bar_mark = '|'
70
+ puts '_' * pbar.send(:get_term_width)
71
+ puts "Class #{klass}: indexing #{total_count} documents in batches of #{batch_size}:\n"
72
+ pbar.send(:show)
73
+
74
+ klass.find_in_batches(:batch_size => batch_size) do |array|
75
+ opts = {:index => index}.merge(options)
76
+ result = Flex.import_collection(array, opts) || next
77
+ f = result.failed
78
+ failed += f
79
+ successful_count += result.successful.count
80
+ pbar.inc(array.size)
81
+ end
82
+
83
+ pbar.finish
84
+ puts "Processed #{total_count}. Successful #{successful_count}. Skipped #{total_count - successful_count - failed.size}. Failed #{failed.size}."
85
+
86
+ if failed.size > 0
87
+ puts 'Failed imports:'
88
+ puts failed.to_yaml
89
+ end
90
+ end
91
+ end
92
+
93
+ private
94
+
95
+ def indices
96
+ indices = ENV['INDEX'] || ENV['INDICES'] || struct.keys
97
+ indices = eval(indices) if indices.is_a?(String)
98
+ indices = [indices] unless indices.is_a?(Array)
99
+ indices
100
+ end
101
+
102
+ def exist?(index)
103
+ Flex.exist?(:index => index)
104
+ end
105
+
106
+ def struct
107
+ @struct ||= begin
108
+ @indices_yaml = ENV['CONFIG_FILE'] || Flex::Configuration.config_file
109
+ raise Errno::ENOENT, "no such file or directory #{@indices_yaml.inspect}. " +
110
+ 'Please, use CONFIG_FILE=/path/to/index.yml ' +
111
+ 'or set the Flex::Configuration.config_file properly' \
112
+ unless File.exist?(@indices_yaml)
113
+ Model::Manager.indices(@indices_yaml)
114
+ end
115
+ end
116
+
117
+
118
+ def models
119
+ @models ||= begin
120
+ mods = ENV['MODEL'] || ENV['MODELS'] || Flex::Configuration.flex_models
121
+ raise AgrumentError, 'no class defined. Please use MODEL=AClass or MODELS=[ClassA,ClassB]' +
122
+ 'or set the Flex::Configuration.flex_models properly' \
123
+ if mods.nil? || mods.empty?
124
+ mods = eval(mods) if mods.is_a?(String)
125
+ mods = [mods] unless mods.is_a?(Array)
126
+ mods.map{|c| c.is_a?(String) ? eval("::#{c}") : c}
127
+ end
128
+ end
129
+
130
+ def delete_index(index)
131
+ Flex.delete_index(:index => index) if exist?(index)
132
+ end
133
+
134
+ def create(index)
135
+ raise MissingIndexEntryError, "no #{index.inspect} entry defined in #@indices_yaml" \
136
+ unless struct.has_key?(index)
137
+ Flex.POST "/#{index}", struct[index]
138
+ end
139
+
140
+ end
141
+ end
@@ -0,0 +1,187 @@
1
+ module Flex
2
+ # Generic Flex::Template object.
3
+ # This class represents a generic Flex::Template object. It is used as the base class by all the more specific Flex::Template::* classes.
4
+ # You usually don't need to instantiate this class manually, since it is usually used internally.
5
+ # For more details about templates, see the documentation.
6
+ class Template
7
+
8
+ include Base
9
+
10
+ def self.variables
11
+ Variables.new
12
+ end
13
+
14
+ attr_reader :method, :path, :data, :variables, :tags, :partials
15
+
16
+ def initialize(method, path, data=nil, vars=nil)
17
+ @method = method.to_s.upcase
18
+ raise ArgumentError, "#@method method not supported" \
19
+ unless %w[HEAD GET PUT POST DELETE].include?(@method)
20
+ @path = path =~ /^\// ? path : "/#{path}"
21
+ @data = Utils.data_from_source(data)
22
+ @instance_vars = vars
23
+ end
24
+
25
+ def setup(host_flex, name=nil, source_vars=nil)
26
+ @host_flex = host_flex
27
+ @name = name
28
+ @source_vars = source_vars
29
+ self
30
+ end
31
+
32
+ def render(vars={})
33
+ do_render(vars) do |response, int|
34
+ Result.new(self, int[:vars], response)
35
+ end
36
+ end
37
+
38
+ def to_a(vars={})
39
+ int = interpolate(vars)
40
+ a = [method, int[:path], int[:data], @instance_vars]
41
+ 2.times { a.pop if a.last.nil? }
42
+ a
43
+ end
44
+
45
+ def to_curl(vars={})
46
+ to_curl_string interpolate(vars, strict=true)
47
+ end
48
+
49
+ def to_flex(name=nil)
50
+ (name ? {name.to_s => to_a} : to_a).to_yaml
51
+ end
52
+
53
+ private
54
+
55
+ def do_render(vars={})
56
+ int = interpolate(vars, strict=true)
57
+ path = build_path(int, vars)
58
+ encoded_data = build_data(int, vars)
59
+ response = Configuration.http_client.request(method, path, encoded_data)
60
+
61
+ # used in Flex.exist?
62
+ return response.status == 200 if method == 'HEAD'
63
+
64
+ if Configuration.raise_proc.call(response)
65
+ int[:vars][:raise].is_a?(FalseClass) ? return : raise(HttpError.new(response, caller_line))
66
+ end
67
+
68
+ result = yield(response, int)
69
+
70
+ rescue NameError => e
71
+ if e.name == :request
72
+ raise MissingHttpClientError, 'you should install the gem "patron" (recommended for performances) or "rest-client", ' +
73
+ 'or provide your own http-client interface and set Flex::Configuration.http_client'
74
+ else
75
+ raise
76
+ end
77
+ ensure
78
+ to_logger(path, encoded_data, result) if int && Configuration.debug && Configuration.logger.level == 0
79
+ end
80
+
81
+ def build_path(int, vars)
82
+ params = int[:vars][:params]
83
+ path = vars[:path] || int[:path]
84
+ if params
85
+ path << ((path =~ /\?/) ? '&' : '?')
86
+ path << params.map { |p| p.join('=') }.join('&')
87
+ end
88
+ path
89
+ end
90
+
91
+ def build_data(int, vars)
92
+ data = vars[:data] && Utils.data_from_source(vars[:data]) || int[:data]
93
+ (data.nil? || data.is_a?(String)) ? data : MultiJson.encode(data)
94
+ end
95
+
96
+ def to_logger(path, encoded_data, result)
97
+ h = {}
98
+ h[:method] = method
99
+ h[:path] = path
100
+ h[:data] = MultiJson.decode(encoded_data) unless encoded_data.nil?
101
+ h[:result] = result if result && Configuration.debug_result
102
+ log = Configuration.debug_to_curl ? to_curl_string(h) : Utils.stringified_hash(h).to_yaml
103
+ Configuration.logger.debug "[FLEX] Rendered #{caller_line}\n#{log}"
104
+ end
105
+
106
+ def caller_line
107
+ method_name = @host_flex && @name && "#{@host_flex.host_class}.#@name"
108
+ line = caller.find{|l| l !~ %r|/flex/lib/flex/[^\.]+\.rb|}
109
+ ll = ''
110
+ ll << "#{method_name} from " if method_name
111
+ ll << "#{line}"
112
+ ll
113
+ end
114
+
115
+ def to_curl_string(h)
116
+ pretty = h[:path] =~ /\?/ ? '&pretty=1' : '?pretty=1'
117
+ curl = %(curl -X#{method} "#{Configuration.base_uri}#{h[:path]}#{pretty}")
118
+ if h[:data]
119
+ data = if h[:data].is_a?(String)
120
+ h[:data].length > 1024 ? h[:data][0,1024] + '...(truncated display)' : h[:data]
121
+ else
122
+ MultiJson.encode(h[:data], :pretty => true)
123
+ end
124
+ curl << %( -d '\n#{data}\n')
125
+ end
126
+ curl
127
+ end
128
+
129
+ def interpolate(*args)
130
+ tags = Tags.new
131
+ stringified = tags.stringify(:path => @path, :data => @data)
132
+ @partials, @tags = tags.map(&:name).partition{|n| n.to_s =~ /^_/}
133
+ @variables = Configuration.variables.deep_dup
134
+ @variables.add(self.class.variables)
135
+ @variables.add(@host_flex.variables) if @host_flex
136
+ @variables.add(@source_vars, @instance_vars, tags.variables)
137
+ instance_eval <<-ruby, __FILE__, __LINE__
138
+ def interpolate(vars={}, strict=false)
139
+ return {:path => path, :data => data, :vars => vars} if vars.empty? && !strict
140
+ sym_vars = {}
141
+ vars.each{|k,v| sym_vars[k.to_sym] = v} # so you can pass the rails params hash
142
+ merged = @variables.deep_merge sym_vars
143
+ vars = process_vars(merged)
144
+ obj = #{stringified}
145
+ obj[:vars] = vars
146
+ obj = prune(obj, vars[:no_pruning])
147
+ obj
148
+ end
149
+ ruby
150
+ interpolate(*args)
151
+ end
152
+
153
+ # prunes the branch when the leaf is nil
154
+ # also removes empty path segments
155
+ # and compact.flatten the Array values
156
+ def prune(obj, no_pruning)
157
+ case obj
158
+ when Array
159
+ return if obj.empty?
160
+ a = obj.map do |i|
161
+ next if i.nil?
162
+ prune(i, no_pruning)
163
+ end.compact.flatten
164
+ a unless a.empty?
165
+ when Hash
166
+ return if obj.empty?
167
+ h = {}
168
+ obj.each do |k, v|
169
+ next if v.nil?
170
+ v = case
171
+ when k == :path
172
+ v.gsub(/\/+/, '/')
173
+ when no_pruning.include?(k)
174
+ v
175
+ else
176
+ prune(v, no_pruning)
177
+ end
178
+ h[k] = v unless v.nil?
179
+ end
180
+ h unless h.empty?
181
+ else
182
+ obj
183
+ end
184
+ end
185
+
186
+ end
187
+ end