load_balanced_tire 0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (121) hide show
  1. data/.gitignore +14 -0
  2. data/.travis.yml +29 -0
  3. data/Gemfile +4 -0
  4. data/MIT-LICENSE +20 -0
  5. data/README.markdown +760 -0
  6. data/Rakefile +78 -0
  7. data/examples/rails-application-template.rb +249 -0
  8. data/examples/tire-dsl.rb +876 -0
  9. data/lib/tire.rb +55 -0
  10. data/lib/tire/alias.rb +296 -0
  11. data/lib/tire/configuration.rb +30 -0
  12. data/lib/tire/dsl.rb +43 -0
  13. data/lib/tire/http/client.rb +62 -0
  14. data/lib/tire/http/clients/curb.rb +61 -0
  15. data/lib/tire/http/clients/faraday.rb +71 -0
  16. data/lib/tire/http/response.rb +27 -0
  17. data/lib/tire/index.rb +361 -0
  18. data/lib/tire/logger.rb +60 -0
  19. data/lib/tire/model/callbacks.rb +40 -0
  20. data/lib/tire/model/import.rb +26 -0
  21. data/lib/tire/model/indexing.rb +128 -0
  22. data/lib/tire/model/naming.rb +100 -0
  23. data/lib/tire/model/percolate.rb +99 -0
  24. data/lib/tire/model/persistence.rb +71 -0
  25. data/lib/tire/model/persistence/attributes.rb +143 -0
  26. data/lib/tire/model/persistence/finders.rb +66 -0
  27. data/lib/tire/model/persistence/storage.rb +69 -0
  28. data/lib/tire/model/search.rb +307 -0
  29. data/lib/tire/results/collection.rb +114 -0
  30. data/lib/tire/results/item.rb +86 -0
  31. data/lib/tire/results/pagination.rb +54 -0
  32. data/lib/tire/rubyext/hash.rb +8 -0
  33. data/lib/tire/rubyext/ruby_1_8.rb +7 -0
  34. data/lib/tire/rubyext/symbol.rb +11 -0
  35. data/lib/tire/search.rb +188 -0
  36. data/lib/tire/search/facet.rb +74 -0
  37. data/lib/tire/search/filter.rb +28 -0
  38. data/lib/tire/search/highlight.rb +37 -0
  39. data/lib/tire/search/query.rb +186 -0
  40. data/lib/tire/search/scan.rb +114 -0
  41. data/lib/tire/search/script_field.rb +23 -0
  42. data/lib/tire/search/sort.rb +25 -0
  43. data/lib/tire/tasks.rb +135 -0
  44. data/lib/tire/utils.rb +17 -0
  45. data/lib/tire/version.rb +22 -0
  46. data/test/fixtures/articles/1.json +1 -0
  47. data/test/fixtures/articles/2.json +1 -0
  48. data/test/fixtures/articles/3.json +1 -0
  49. data/test/fixtures/articles/4.json +1 -0
  50. data/test/fixtures/articles/5.json +1 -0
  51. data/test/integration/active_model_indexing_test.rb +51 -0
  52. data/test/integration/active_model_searchable_test.rb +114 -0
  53. data/test/integration/active_record_searchable_test.rb +446 -0
  54. data/test/integration/boolean_queries_test.rb +43 -0
  55. data/test/integration/count_test.rb +34 -0
  56. data/test/integration/custom_score_queries_test.rb +88 -0
  57. data/test/integration/dis_max_queries_test.rb +68 -0
  58. data/test/integration/dsl_search_test.rb +22 -0
  59. data/test/integration/explanation_test.rb +44 -0
  60. data/test/integration/facets_test.rb +259 -0
  61. data/test/integration/filtered_queries_test.rb +66 -0
  62. data/test/integration/filters_test.rb +63 -0
  63. data/test/integration/fuzzy_queries_test.rb +20 -0
  64. data/test/integration/highlight_test.rb +64 -0
  65. data/test/integration/index_aliases_test.rb +122 -0
  66. data/test/integration/index_mapping_test.rb +43 -0
  67. data/test/integration/index_store_test.rb +96 -0
  68. data/test/integration/index_update_document_test.rb +111 -0
  69. data/test/integration/mongoid_searchable_test.rb +309 -0
  70. data/test/integration/percolator_test.rb +111 -0
  71. data/test/integration/persistent_model_test.rb +130 -0
  72. data/test/integration/prefix_query_test.rb +43 -0
  73. data/test/integration/query_return_version_test.rb +70 -0
  74. data/test/integration/query_string_test.rb +52 -0
  75. data/test/integration/range_queries_test.rb +36 -0
  76. data/test/integration/reindex_test.rb +46 -0
  77. data/test/integration/results_test.rb +39 -0
  78. data/test/integration/scan_test.rb +56 -0
  79. data/test/integration/script_fields_test.rb +38 -0
  80. data/test/integration/sort_test.rb +36 -0
  81. data/test/integration/text_query_test.rb +39 -0
  82. data/test/models/active_model_article.rb +31 -0
  83. data/test/models/active_model_article_with_callbacks.rb +49 -0
  84. data/test/models/active_model_article_with_custom_document_type.rb +7 -0
  85. data/test/models/active_model_article_with_custom_index_name.rb +7 -0
  86. data/test/models/active_record_models.rb +122 -0
  87. data/test/models/article.rb +15 -0
  88. data/test/models/mongoid_models.rb +97 -0
  89. data/test/models/persistent_article.rb +11 -0
  90. data/test/models/persistent_article_in_namespace.rb +12 -0
  91. data/test/models/persistent_article_with_casting.rb +28 -0
  92. data/test/models/persistent_article_with_defaults.rb +11 -0
  93. data/test/models/persistent_articles_with_custom_index_name.rb +10 -0
  94. data/test/models/supermodel_article.rb +17 -0
  95. data/test/models/validated_model.rb +11 -0
  96. data/test/test_helper.rb +93 -0
  97. data/test/unit/active_model_lint_test.rb +17 -0
  98. data/test/unit/configuration_test.rb +74 -0
  99. data/test/unit/http_client_test.rb +76 -0
  100. data/test/unit/http_response_test.rb +49 -0
  101. data/test/unit/index_alias_test.rb +275 -0
  102. data/test/unit/index_test.rb +894 -0
  103. data/test/unit/logger_test.rb +125 -0
  104. data/test/unit/model_callbacks_test.rb +116 -0
  105. data/test/unit/model_import_test.rb +71 -0
  106. data/test/unit/model_persistence_test.rb +528 -0
  107. data/test/unit/model_search_test.rb +913 -0
  108. data/test/unit/results_collection_test.rb +281 -0
  109. data/test/unit/results_item_test.rb +162 -0
  110. data/test/unit/rubyext_test.rb +66 -0
  111. data/test/unit/search_facet_test.rb +153 -0
  112. data/test/unit/search_filter_test.rb +42 -0
  113. data/test/unit/search_highlight_test.rb +46 -0
  114. data/test/unit/search_query_test.rb +301 -0
  115. data/test/unit/search_scan_test.rb +113 -0
  116. data/test/unit/search_script_field_test.rb +26 -0
  117. data/test/unit/search_sort_test.rb +50 -0
  118. data/test/unit/search_test.rb +499 -0
  119. data/test/unit/tire_test.rb +126 -0
  120. data/tire.gemspec +90 -0
  121. metadata +549 -0
@@ -0,0 +1,60 @@
1
+ module Tire
2
+ class Logger
3
+
4
+ def initialize(device, options={})
5
+ @device = if device.respond_to?(:write)
6
+ device
7
+ else
8
+ File.open(device, 'a')
9
+ end
10
+ @device.sync = true if @device.respond_to?(:sync)
11
+ @options = options
12
+ # at_exit { @device.close unless @device.closed? } if @device.respond_to?(:closed?) && @device.respond_to?(:close)
13
+ end
14
+
15
+ def level
16
+ @options[:level] || 'info'
17
+ end
18
+
19
+ def write(message)
20
+ @device.write message
21
+ end
22
+
23
+ def log_request(endpoint, params=nil, curl='')
24
+ # 2001-02-12 18:20:42:32 [_search] (articles,users)
25
+ #
26
+ # curl -X POST ....
27
+ #
28
+ content = "# #{time}"
29
+ content += " [#{endpoint}]"
30
+ content += " (#{params.inspect})" if params
31
+ content += "\n#\n"
32
+ content += curl
33
+ content += "\n\n"
34
+ write content
35
+ end
36
+
37
+ def log_response(status, took=nil, json='')
38
+ # 2001-02-12 18:20:42:32 [200] (4 msec)
39
+ #
40
+ # {
41
+ # "took" : 4,
42
+ # "hits" : [...]
43
+ # ...
44
+ # }
45
+ #
46
+ content = "# #{time}"
47
+ content += " [#{status}]"
48
+ content += " (#{took} msec)" if took
49
+ content += "\n#\n" unless json.to_s !~ /\S/
50
+ json.to_s.each_line { |line| content += "# #{line}" } unless json.to_s !~ /\S/
51
+ content += "\n\n"
52
+ write content
53
+ end
54
+
55
+ def time
56
+ Time.now.strftime('%Y-%m-%d %H:%M:%S:%L')
57
+ end
58
+
59
+ end
60
+ end
@@ -0,0 +1,40 @@
1
+ module Tire
2
+ module Model
3
+
4
+ # Main module containing the infrastructure for automatic updating
5
+ # of the _ElasticSearch_ index on model instance create, update or delete.
6
+ #
7
+ # Include it in your model: `include Tire::Model::Callbacks`
8
+ #
9
+ # The model must respond to `after_save` and `after_destroy` callbacks
10
+ # (ActiveModel and ActiveRecord models do so, by default).
11
+ #
12
+ module Callbacks
13
+
14
+ # A hook triggered by the `include Tire::Model::Callbacks` statement in the model.
15
+ #
16
+ def self.included(base)
17
+
18
+ # Update index on model instance change or destroy.
19
+ #
20
+ if base.respond_to?(:after_save) && base.respond_to?(:after_destroy)
21
+ base.send :after_save, lambda { tire.update_index }
22
+ base.send :after_destroy, lambda { tire.update_index }
23
+ end
24
+
25
+ # Add neccessary infrastructure for the model, when missing in
26
+ # some half-baked ActiveModel implementations.
27
+ #
28
+ if base.respond_to?(:before_destroy) && !base.instance_methods.map(&:to_sym).include?(:destroyed?)
29
+ base.class_eval do
30
+ before_destroy { @destroyed = true }
31
+ def destroyed?; !!@destroyed; end
32
+ end
33
+ end
34
+
35
+ end
36
+
37
+ end
38
+
39
+ end
40
+ end
@@ -0,0 +1,26 @@
1
+
2
+ module Tire
3
+ module Model
4
+
5
+ # Provides support for easy importing of large ActiveRecord- and ActiveModel-bound
6
+ # recordsets into model index.
7
+ #
8
+ # Relies on pagination support in your model, namely the `paginate` class method.
9
+ #
10
+ # Please refer to the relevant of the README for more information.
11
+ #
12
+ module Import
13
+
14
+ module ClassMethods
15
+
16
+ def import options={}, &block
17
+ options = { :method => 'paginate' }.update options
18
+ index.import klass, options, &block
19
+ end
20
+
21
+ end
22
+
23
+ end
24
+
25
+ end
26
+ end
@@ -0,0 +1,128 @@
1
+ module Tire
2
+ module Model
3
+
4
+ # Contains logic for definition of index settings and mappings.
5
+ #
6
+ module Indexing
7
+
8
+ module ClassMethods
9
+
10
+ # Define [_settings_](http://www.elasticsearch.org/guide/reference/api/admin-indices-create-index.html)
11
+ # for the corresponding index, such as number of shards and replicas, custom analyzers, etc.
12
+ #
13
+ # Usage:
14
+ #
15
+ # class Article
16
+ # # ...
17
+ # settings :number_of_shards => 1 do
18
+ # mapping do
19
+ # # ...
20
+ # end
21
+ # end
22
+ # end
23
+ #
24
+ def settings(*args)
25
+ @settings ||= {}
26
+ args.empty? ? (return @settings) : @settings = args.pop
27
+ yield if block_given?
28
+ end
29
+
30
+ # Define the [_mapping_](http://www.elasticsearch.org/guide/reference/mapping/index.html)
31
+ # for the corresponding index, telling _ElasticSearch_ how to understand your documents:
32
+ # what type is which property, whether it is analyzed or no, which analyzer to use, etc.
33
+ #
34
+ # You may pass the top level mapping properties (such as `_source` or `_all`) as a Hash.
35
+ #
36
+ # Usage:
37
+ #
38
+ # class Article
39
+ # # ...
40
+ # mapping :_source => { :compress => true } do
41
+ # indexes :id, :index => :not_analyzed
42
+ # indexes :title, :analyzer => 'snowball', :boost => 100
43
+ # indexes :words, :as => 'content.split(/\W/).length'
44
+ #
45
+ # indexes :comments do
46
+ # indexes :body
47
+ # indexes :author do
48
+ # indexes :name
49
+ # end
50
+ # end
51
+ #
52
+ # # ...
53
+ # end
54
+ # end
55
+ #
56
+ def mapping(*args)
57
+ @mapping ||= {}
58
+ if block_given?
59
+ @mapping_options = args.pop
60
+ yield
61
+ create_elasticsearch_index
62
+ else
63
+ @mapping
64
+ end
65
+ end
66
+
67
+ # Define mapping for the property passed as the first argument (`name`)
68
+ # using definition from the second argument (`options`).
69
+ #
70
+ # `:type` is optional and defaults to `'string'`.
71
+ #
72
+ # Usage:
73
+ #
74
+ # * Index property but do not analyze it: `indexes :id, :index => :not_analyzed`
75
+ #
76
+ # * Use different analyzer for indexing a property: `indexes :title, :analyzer => 'snowball'`
77
+ #
78
+ # * Use the `:as` option to dynamically define the serialized property value, eg:
79
+ #
80
+ # :as => 'content.split(/\W/).length'
81
+ #
82
+ # Please refer to the
83
+ # [_mapping_ documentation](http://www.elasticsearch.org/guide/reference/mapping/index.html)
84
+ # for more information.
85
+ #
86
+ def indexes(name, options = {}, &block)
87
+ mapping[name] = options
88
+
89
+ if block_given?
90
+ mapping[name][:type] ||= 'object'
91
+ mapping[name][:properties] ||= {}
92
+
93
+ previous = @mapping
94
+ @mapping = mapping[name][:properties]
95
+ yield
96
+ @mapping = previous
97
+ end
98
+
99
+ mapping[name][:type] ||= 'string'
100
+
101
+ self
102
+ end
103
+
104
+ # Creates the corresponding index with desired settings and mappings, when it does not exists yet.
105
+ #
106
+ def create_elasticsearch_index
107
+ unless index.exists?
108
+ index.create :mappings => mapping_to_hash, :settings => settings
109
+ end
110
+ rescue Errno::ECONNREFUSED => e
111
+ STDERR.puts "Skipping index creation, cannot connect to ElasticSearch",
112
+ "(The original exception was: #{e.inspect})"
113
+ end
114
+
115
+ def mapping_options
116
+ @mapping_options || {}
117
+ end
118
+
119
+ def mapping_to_hash
120
+ { document_type.to_sym => mapping_options.merge({ :properties => mapping }) }
121
+ end
122
+
123
+ end
124
+
125
+ end
126
+
127
+ end
128
+ end
@@ -0,0 +1,100 @@
1
+ module Tire
2
+ module Model
3
+
4
+ # Contains logic for getting and setting the index name and document type for this model.
5
+ #
6
+ module Naming
7
+
8
+ module ClassMethods
9
+
10
+ # Get or set the index name for this model, based on arguments.
11
+ #
12
+ # By default, uses ActiveSupport inflection, so a class named `Article`
13
+ # will be stored in the `articles` index.
14
+ #
15
+ # To get the index name:
16
+ #
17
+ # Article.index_name
18
+ #
19
+ # To set the index name:
20
+ #
21
+ # Article.index_name 'my-custom-name'
22
+ #
23
+ # You can also use a block for defining the index name,
24
+ # which is evaluated in the class context:
25
+ #
26
+ # Article.index_name { "articles-#{Time.now.year}" }
27
+ #
28
+ # Article.index_name { "articles-#{Rails.env}" }
29
+ #
30
+ def index_name name=nil, &block
31
+ @index_name = name if name
32
+ @index_name = block if block_given?
33
+ # TODO: Try to get index_name from ancestor classes
34
+ @index_name || [index_prefix, klass.model_name.plural].compact.join('_')
35
+ end
36
+
37
+ # Set or get index prefix for all models or for a specific model.
38
+ #
39
+ # To set the prefix for all models (preferably in an initializer inside Rails):
40
+ #
41
+ # Tire::Model::Search.index_prefix Rails.env
42
+ #
43
+ # To set the prefix for specific model:
44
+ #
45
+ # class Article
46
+ # # ...
47
+ # index_prefix 'my_prefix'
48
+ # end
49
+ #
50
+ # TODO: Maybe this would be more sane with ActiveSupport extensions such as `class_attribute`?
51
+ #
52
+ @@__index_prefix__ = nil
53
+ def index_prefix(*args)
54
+ # Uses class or instance variable depending on the context
55
+ if args.size > 0
56
+ value = args.pop
57
+ self.is_a?(Module) ? ( @@__index_prefix__ = value ) : ( @__index_prefix__ = value )
58
+ end
59
+ self.is_a?(Module) ? ( @@__index_prefix__ || nil ) : ( @__index_prefix__ || @@__index_prefix__ || nil )
60
+ end
61
+ extend self
62
+
63
+ # Get or set the document type for this model, based on arguments.
64
+ #
65
+ # By default, uses ActiveSupport inflection, so a class named `Article`
66
+ # will be stored as the `article` type.
67
+ #
68
+ # To get the document type:
69
+ #
70
+ # Article.document_type
71
+ #
72
+ # To set the document type:
73
+ #
74
+ # Article.document_type 'my-custom-type'
75
+ #
76
+ def document_type name=nil
77
+ @document_type = name if name
78
+ @document_type || klass.model_name.underscore
79
+ end
80
+ end
81
+
82
+ module InstanceMethods
83
+
84
+ # Proxy to class method `index_name`.
85
+ #
86
+ def index_name
87
+ instance.class.tire.index_name
88
+ end
89
+
90
+ # Proxy to instance method `document_type`.
91
+ #
92
+ def document_type
93
+ instance.class.tire.document_type
94
+ end
95
+ end
96
+
97
+ end
98
+
99
+ end
100
+ end
@@ -0,0 +1,99 @@
1
+ module Tire
2
+ module Model
3
+
4
+ # Contains support for the [percolation](http://www.elasticsearch.org/guide/reference/api/percolate.html)
5
+ # feature of _ElasticSearch_.
6
+ #
7
+ module Percolate
8
+
9
+ module ClassMethods
10
+
11
+ # Set up the percolation when documents are being added to the index.
12
+ #
13
+ # Usage:
14
+ #
15
+ # class Article
16
+ # # ...
17
+ # percolate!
18
+ # end
19
+ #
20
+ # First, you have to register a percolator query:
21
+ #
22
+ # Article.index.register_percolator_query('fail') { |query| query.string 'fail' }
23
+ #
24
+ # Then, when you update the index, matching queries are returned in the `matches` property:
25
+ #
26
+ # p Article.create(:title => 'This is a FAIL!').matches
27
+ #
28
+ #
29
+ # You may pass a pattern to filter which percolator queries will be executed.
30
+ #
31
+ # See <http://www.elasticsearch.org/guide/reference/api/index_.html> for more information.
32
+ #
33
+ def percolate!(pattern=true)
34
+ @@_percolator = pattern
35
+ self
36
+ end
37
+
38
+ # A callback method for intercepting percolator matches.
39
+ #
40
+ # Usage:
41
+ #
42
+ # class Article
43
+ # # ...
44
+ # on_percolate do
45
+ # puts "Article title “#{title}” matches queries: #{matches.inspect}" unless matches.empty?
46
+ # end
47
+ # end
48
+ #
49
+ # Based on the response received in `matches`, you may choose to fire notifications,
50
+ # increment counters, send out e-mail alerts, etc.
51
+ #
52
+ def on_percolate(pattern=true,&block)
53
+ percolate!(pattern)
54
+ klass.after_update_elasticsearch_index(block)
55
+ end
56
+
57
+ # Returns the status or pattern of percolator for this class.
58
+ #
59
+ def percolator
60
+ defined?(@@_percolator) ? @@_percolator : nil
61
+ end
62
+ end
63
+
64
+ module InstanceMethods
65
+
66
+ # Run this document against registered percolator queries, without indexing it.
67
+ #
68
+ # First, register a percolator query:
69
+ #
70
+ # Article.index.register_percolator_query('fail') { |query| query.string 'fail' }
71
+ #
72
+ # Then, you may query the percolator endpoint with:
73
+ #
74
+ # p Article.new(:title => 'This is a FAIL!').percolate
75
+ #
76
+ # Optionally, you may pass a block to filter which percolator queries will be executed.
77
+ #
78
+ # See <http://www.elasticsearch.org/guide/reference/api/percolate.html> for more information.
79
+ def percolate(&block)
80
+ index.percolate instance, block
81
+ end
82
+
83
+ # Mark this instance for percolation when adding it to the index.
84
+ #
85
+ def percolate=(pattern)
86
+ @_percolator = pattern
87
+ end
88
+
89
+ # Returns the status or pattern of percolator for this instance.
90
+ #
91
+ def percolator
92
+ @_percolator || instance.class.tire.percolator || nil
93
+ end
94
+ end
95
+
96
+ end
97
+
98
+ end
99
+ end