indexers 4.1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +7 -0
  2. data/MIT-LICENSE +20 -0
  3. data/README.md +313 -0
  4. data/Rakefile +19 -0
  5. data/lib/generators/indexers/indexer/indexer_generator.rb +23 -0
  6. data/lib/generators/indexers/indexer/templates/indexer.rb +12 -0
  7. data/lib/generators/indexers/install/install_generator.rb +19 -0
  8. data/lib/generators/indexers/install/templates/configuration.yml +15 -0
  9. data/lib/generators/indexers/install/templates/initializer.rb +6 -0
  10. data/lib/indexers/collection.rb +181 -0
  11. data/lib/indexers/computed_sorts.rb +19 -0
  12. data/lib/indexers/concern.rb +30 -0
  13. data/lib/indexers/configuration.rb +35 -0
  14. data/lib/indexers/definitions.rb +24 -0
  15. data/lib/indexers/dsl/api.rb +94 -0
  16. data/lib/indexers/dsl/mappings.rb +14 -0
  17. data/lib/indexers/dsl/search.rb +29 -0
  18. data/lib/indexers/dsl/serialization.rb +17 -0
  19. data/lib/indexers/dsl/traitable.rb +38 -0
  20. data/lib/indexers/extensions/active_record/base.rb +20 -0
  21. data/lib/indexers/indexer.rb +132 -0
  22. data/lib/indexers/pagination.rb +33 -0
  23. data/lib/indexers/proxy.rb +22 -0
  24. data/lib/indexers/railtie.rb +23 -0
  25. data/lib/indexers/version.rb +5 -0
  26. data/lib/indexers.rb +82 -0
  27. data/lib/tasks/indexers.rake +16 -0
  28. data/test/dsl_test.rb +127 -0
  29. data/test/dummy/Rakefile +5 -0
  30. data/test/dummy/app/assets/javascripts/application.js +13 -0
  31. data/test/dummy/app/assets/stylesheets/application.css +15 -0
  32. data/test/dummy/app/controllers/application_controller.rb +5 -0
  33. data/test/dummy/app/helpers/application_helper.rb +2 -0
  34. data/test/dummy/app/indexers/product_indexer.rb +55 -0
  35. data/test/dummy/app/indexers/shop_indexer.rb +28 -0
  36. data/test/dummy/app/models/product.rb +5 -0
  37. data/test/dummy/app/models/shop.rb +5 -0
  38. data/test/dummy/app/views/layouts/application.html.erb +12 -0
  39. data/test/dummy/bin/bundle +4 -0
  40. data/test/dummy/bin/rails +5 -0
  41. data/test/dummy/bin/rake +5 -0
  42. data/test/dummy/bin/setup +30 -0
  43. data/test/dummy/config/application.rb +25 -0
  44. data/test/dummy/config/boot.rb +5 -0
  45. data/test/dummy/config/database.yml +10 -0
  46. data/test/dummy/config/database.yml.travis +3 -0
  47. data/test/dummy/config/elasticsearch.yml +15 -0
  48. data/test/dummy/config/environment.rb +5 -0
  49. data/test/dummy/config/environments/development.rb +41 -0
  50. data/test/dummy/config/environments/production.rb +79 -0
  51. data/test/dummy/config/environments/test.rb +42 -0
  52. data/test/dummy/config/initializers/assets.rb +11 -0
  53. data/test/dummy/config/initializers/backtrace_silencers.rb +7 -0
  54. data/test/dummy/config/initializers/cookies_serializer.rb +3 -0
  55. data/test/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  56. data/test/dummy/config/initializers/indexers.rb +65 -0
  57. data/test/dummy/config/initializers/inflections.rb +16 -0
  58. data/test/dummy/config/initializers/mime_types.rb +4 -0
  59. data/test/dummy/config/initializers/session_store.rb +3 -0
  60. data/test/dummy/config/initializers/wrap_parameters.rb +14 -0
  61. data/test/dummy/config/locales/en.yml +23 -0
  62. data/test/dummy/config/routes.rb +56 -0
  63. data/test/dummy/config/secrets.yml +22 -0
  64. data/test/dummy/config.ru +4 -0
  65. data/test/dummy/db/migrate/20161104164148_create_products.rb +14 -0
  66. data/test/dummy/db/migrate/20161104182219_create_shops.rb +9 -0
  67. data/test/dummy/db/schema.rb +36 -0
  68. data/test/dummy/log/development.log +114 -0
  69. data/test/dummy/log/test.log +20986 -0
  70. data/test/dummy/public/404.html +61 -0
  71. data/test/dummy/public/422.html +61 -0
  72. data/test/dummy/public/500.html +60 -0
  73. data/test/dummy/public/favicon.ico +0 -0
  74. data/test/generator_test.rb +26 -0
  75. data/test/index_test.rb +42 -0
  76. data/test/record_test.rb +25 -0
  77. data/test/search_test.rb +164 -0
  78. data/test/task_test.rb +31 -0
  79. data/test/test_helper.rb +14 -0
  80. metadata +237 -0
@@ -0,0 +1,24 @@
1
+ module Indexers
2
+ class Definitions
3
+
4
+ def add(*args)
5
+ indexer = Indexer.new(*args)
6
+ registry[indexer.name] = indexer
7
+ end
8
+
9
+ def find(name)
10
+ registry[name]
11
+ end
12
+
13
+ def each(&block)
14
+ registry.values.sort.each &block
15
+ end
16
+
17
+ private
18
+
19
+ def registry
20
+ @registry ||= {}
21
+ end
22
+
23
+ end
24
+ end
@@ -0,0 +1,94 @@
1
+ module Indexers
2
+ module Dsl
3
+ class Api
4
+
5
+ def initialize(args=[], parent={}, &block)
6
+ @parent = parent
7
+ instance_exec *args, &block
8
+ end
9
+
10
+ def method_missing(name, *args, &block)
11
+ options = args.extract_options!
12
+ name = name.to_sym
13
+ if block_given?
14
+ add_block name, args, options, &block
15
+ elsif args.size > 0
16
+ add_argument name, args, options
17
+ elsif options.any?
18
+ add_options name, options
19
+ else
20
+ add_empty name
21
+ end
22
+ end
23
+
24
+ def to_h
25
+ @parent
26
+ end
27
+
28
+ private
29
+
30
+ def add_block(name, args, options, &block)
31
+ case args.first
32
+ when String,Symbol
33
+ child = {}
34
+ node = { args.first.to_sym => child }
35
+ when Enumerable,ActiveRecord::Relation
36
+ child = node = []
37
+ else
38
+ child = node = {}
39
+ end
40
+ case @parent
41
+ when Array
42
+ @parent << options.merge(name => node)
43
+ when Hash
44
+ @parent[name] = node
45
+ end
46
+ case args.first
47
+ when Enumerable,ActiveRecord::Relation
48
+ args.first.each do |arg|
49
+ continue [arg], child, &block
50
+ end
51
+ else
52
+ continue [], child, &block
53
+ end
54
+ end
55
+
56
+ def add_argument(name, args, options)
57
+ case @parent
58
+ when Array
59
+ @parent << { name => args.first }
60
+ when Hash
61
+ @parent[name] = args.first
62
+ end
63
+ end
64
+
65
+ def add_options(name, options)
66
+ options.symbolize_keys!
67
+ case @parent
68
+ when Array
69
+ @parent << { name => options }
70
+ when Hash
71
+ if @parent.has_key?(name)
72
+ @parent[name].merge! options
73
+ else
74
+ @parent[name] = options
75
+ end
76
+ end
77
+ end
78
+
79
+ def add_empty(name)
80
+ case @parent
81
+ when Array
82
+ @parent << { name => {} }
83
+ when Hash
84
+ @parent[name] = {}
85
+ end
86
+ end
87
+
88
+ def continue(args, parent, &block)
89
+ self.class.new args, parent, &block
90
+ end
91
+
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,14 @@
1
+ module Indexers
2
+ module Dsl
3
+ class Mappings < Api
4
+
5
+ def properties(*names)
6
+ @parent[:properties] ||= {}
7
+ names.each do |name|
8
+ @parent[:properties][name] = Indexers.configuration.mappings[name]
9
+ end
10
+ end
11
+
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,29 @@
1
+ module Indexers
2
+ module Dsl
3
+ class Search < Traitable
4
+
5
+ private
6
+
7
+ def add_block(name, args, options, &block)
8
+ if %i(functions must must_not should).include?(name)
9
+ child = []
10
+ @parent[name] = child
11
+ continue [], child, &block
12
+ else
13
+ super
14
+ end
15
+ end
16
+
17
+ def add_argument(name, args, options)
18
+ if name == :query && args.any?
19
+ indexer = Indexers.definitions.find(args.first)
20
+ hash = self.class.new(indexer, [options], &indexer.options[:search]).to_h
21
+ @parent[name] = hash[:query]
22
+ else
23
+ super
24
+ end
25
+ end
26
+
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,17 @@
1
+ module Indexers
2
+ module Dsl
3
+ class Serialization < Traitable
4
+
5
+ def extract(record, *names)
6
+ names.each do |name|
7
+ send name, record.send(name)
8
+ end
9
+ end
10
+
11
+ def transliterate(value)
12
+ ActiveSupport::Inflector.transliterate value
13
+ end
14
+
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,38 @@
1
+ module Indexers
2
+ module Dsl
3
+ class Traitable < Api
4
+
5
+ def initialize(indexer=nil, args=[], parent={}, binding=nil, &block)
6
+ @indexer = indexer
7
+ @binding = binding
8
+ @block = block
9
+ super args, parent, &block
10
+ end
11
+
12
+ def traits(*names)
13
+ if @indexer
14
+ @binding = @block.binding
15
+ names.each do |name|
16
+ instance_eval &@indexer.options[:traits][name]
17
+ end
18
+ @binding = nil
19
+ end
20
+ end
21
+
22
+ def method_missing(name, *args, &block)
23
+ if args.size == 0 && !block_given? && @binding.try(:local_variable_defined?, name)
24
+ @binding.local_variable_get name
25
+ else
26
+ super
27
+ end
28
+ end
29
+
30
+ private
31
+
32
+ def continue(args, parent, &block)
33
+ self.class.new @indexer, args, parent, @binding, &block
34
+ end
35
+
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,20 @@
1
+ module Indexers
2
+ module Extensions
3
+ module ActiveRecord
4
+ module Base
5
+ extend ActiveSupport::Concern
6
+
7
+ module ClassMethods
8
+
9
+ def inherited(subclass)
10
+ super
11
+ if File.exist?("#{Rails.root}/app/indexers/#{subclass.name.underscore}_indexer.rb")
12
+ subclass.include Indexers::Concern
13
+ end
14
+ end
15
+
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,132 @@
1
+ module Indexers
2
+ class Indexer
3
+
4
+ attr_reader :name, :options
5
+
6
+ def initialize(name, options)
7
+ @name = name
8
+ @options = options
9
+ end
10
+
11
+ def model
12
+ options.fetch(:class_name, name.to_s.classify).constantize
13
+ end
14
+
15
+ def mappings
16
+ @mappings ||= Dsl::Mappings.new(&options[:mappings]).to_h
17
+ end
18
+
19
+ def has_parent?
20
+ mappings.has_key? :_parent
21
+ end
22
+
23
+ def <=>(other)
24
+ if has_parent? && other.has_parent?
25
+ 0
26
+ elsif other.has_parent?
27
+ 1
28
+ else
29
+ -1
30
+ end
31
+ end
32
+
33
+ def any?(*args)
34
+ search(*args).count > 0
35
+ end
36
+
37
+ def none?(*args)
38
+ !any?(*args)
39
+ end
40
+
41
+ def search(query)
42
+ client.search(
43
+ index: namespace,
44
+ type: name,
45
+ body: query
46
+ )
47
+ end
48
+
49
+ def exists?(record)
50
+ client.exists?(
51
+ with_parent(
52
+ record,
53
+ index: namespace,
54
+ type: name,
55
+ id: record.id
56
+ )
57
+ )
58
+ end
59
+
60
+ def index(record)
61
+ client.create(
62
+ with_parent(
63
+ record,
64
+ index: namespace,
65
+ type: name,
66
+ id: record.id,
67
+ body: serialize(record)
68
+ )
69
+ )
70
+ end
71
+
72
+ def reindex(record)
73
+ client.bulk(
74
+ index: namespace,
75
+ type: name,
76
+ body: [
77
+ { delete: with_parent(record, _id: record.id) },
78
+ { index: with_parent(record, _id: record.id, data: serialize(record)) }
79
+ ]
80
+ )
81
+ end
82
+
83
+ def unindex(record)
84
+ client.delete(
85
+ with_parent(
86
+ record,
87
+ index: namespace,
88
+ type: name,
89
+ id: record.id
90
+ )
91
+ )
92
+ end
93
+
94
+ def build
95
+ client.indices.put_mapping(
96
+ index: namespace,
97
+ type: name,
98
+ body: mappings
99
+ )
100
+ model.find_in_batches do |records|
101
+ client.bulk(
102
+ index: namespace,
103
+ type: name,
104
+ body: records.map do |record|
105
+ { index: with_parent(record, _id: record.id, data: serialize(record)) }
106
+ end
107
+ )
108
+ end
109
+ end
110
+
111
+ private
112
+
113
+ %i(client namespace).each do |name|
114
+ define_method name do
115
+ Indexers.send name
116
+ end
117
+ end
118
+
119
+ def with_parent(record, hash)
120
+ if has_parent?
121
+ hash.merge parent: record.send("#{mappings[:_parent][:type]}_id")
122
+ else
123
+ hash
124
+ end
125
+ end
126
+
127
+ def serialize(record)
128
+ Dsl::Serialization.new(self, record, &options[:serialize]).to_h
129
+ end
130
+
131
+ end
132
+ end
@@ -0,0 +1,33 @@
1
+ module Indexers
2
+ module Pagination
3
+
4
+ def total_pages
5
+ @total_pages ||= [(total_count.to_f / page_length).ceil, 1].max
6
+ end
7
+
8
+ def previous_page
9
+ @previous_page ||= (current_page > 1 ? (current_page - 1) : nil)
10
+ end
11
+
12
+ def next_page
13
+ @next_page ||= (current_page < total_pages ? (current_page + 1) : nil)
14
+ end
15
+
16
+ def first_page
17
+ 1
18
+ end
19
+
20
+ def last_page
21
+ total_pages
22
+ end
23
+
24
+ def out_of_bounds?
25
+ @out_of_bounds ||= (current_page > total_pages || current_page < first_page)
26
+ end
27
+
28
+ def total_count
29
+ @total_count ||= (response['hits']['total'].to_i - padding)
30
+ end
31
+
32
+ end
33
+ end
@@ -0,0 +1,22 @@
1
+ module Indexers
2
+ class Proxy
3
+
4
+ def initialize(name, options={}, &block)
5
+ @name = name
6
+ @options = options.merge(traits: {})
7
+ instance_eval &block
8
+ Indexers.definitions.add name, @options
9
+ end
10
+
11
+ %i(mappings serialize search).each do |name|
12
+ define_method name do |&block|
13
+ @options[name] = block
14
+ end
15
+ end
16
+
17
+ def trait(name, &block)
18
+ @options[:traits][name] = block
19
+ end
20
+
21
+ end
22
+ end
@@ -0,0 +1,23 @@
1
+ module Indexers
2
+ class Railtie < Rails::Railtie
3
+
4
+ config.before_initialize do
5
+ Dir["#{Rails.root}/app/indexers/**/*_indexer.rb"].each do |file|
6
+ load file
7
+ end
8
+ end
9
+
10
+ initializer 'indexers.active_record' do
11
+ ActiveSupport.on_load :active_record do
12
+ ::ActiveRecord::Base.include(
13
+ Indexers::Extensions::ActiveRecord::Base
14
+ )
15
+ end
16
+ end
17
+
18
+ rake_tasks do
19
+ load 'tasks/indexers.rake'
20
+ end
21
+
22
+ end
23
+ end
@@ -0,0 +1,5 @@
1
+ module Indexers
2
+
3
+ VERSION = '4.1.0.0'
4
+
5
+ end
data/lib/indexers.rb ADDED
@@ -0,0 +1,82 @@
1
+ require 'indexers/dsl/api'
2
+ require 'indexers/dsl/mappings'
3
+ require 'indexers/dsl/traitable'
4
+ require 'indexers/dsl/search'
5
+ require 'indexers/dsl/serialization'
6
+ require 'indexers/extensions/active_record/base'
7
+ require 'indexers/collection'
8
+ require 'indexers/computed_sorts'
9
+ require 'indexers/concern'
10
+ require 'indexers/configuration'
11
+ require 'indexers/definitions'
12
+ require 'indexers/indexer'
13
+ require 'indexers/pagination'
14
+ require 'indexers/proxy'
15
+ require 'indexers/railtie'
16
+ require 'indexers/version'
17
+
18
+ module Indexers
19
+ class << self
20
+
21
+ def namespace
22
+ "#{Rails.application.class.parent_name} #{Rails.env}".parameterize('_')
23
+ end
24
+
25
+ def client
26
+ @client ||= begin
27
+ require 'elasticsearch'
28
+ Elasticsearch::Client.new YAML.load_file("#{Rails.root}/config/elasticsearch.yml")[Rails.env]
29
+ end
30
+ end
31
+
32
+ def configure
33
+ yield configuration
34
+ end
35
+
36
+ def configuration
37
+ @configuration ||= Configuration.new
38
+ end
39
+
40
+ def computed_sorts
41
+ @computed_sorts ||= ComputedSorts.new
42
+ end
43
+
44
+ def definitions
45
+ @definitions ||= Definitions.new
46
+ end
47
+
48
+ def define(*args, &block)
49
+ Proxy.new *args, &block
50
+ end
51
+
52
+ def index
53
+ unless client.indices.exists?(index: namespace)
54
+ client.indices.create(
55
+ index: namespace,
56
+ body: { settings: configuration.analysis }
57
+ )
58
+ end
59
+ definitions.each &:build
60
+ end
61
+
62
+ def reindex
63
+ unindex
64
+ index
65
+ end
66
+
67
+ def unindex
68
+ if client.indices.exists?(index: namespace)
69
+ client.indices.delete index: namespace
70
+ end
71
+ end
72
+
73
+ def suggest(*args)
74
+ response = client.suggest(
75
+ index: namespace,
76
+ body: { suggestions: Dsl::Api.new(args, &configuration.suggestions).to_h }
77
+ )
78
+ response['suggestions'].first['options'].map &:symbolize_keys
79
+ end
80
+
81
+ end
82
+ end
@@ -0,0 +1,16 @@
1
+ namespace :indexers do
2
+ desc 'Index all records.'
3
+ task index: :environment do
4
+ Indexers.index
5
+ end
6
+
7
+ desc 'Reindex all records.'
8
+ task reindex: :environment do
9
+ Indexers.reindex
10
+ end
11
+
12
+ desc 'Unindex all records.'
13
+ task unindex: :environment do
14
+ Indexers.unindex
15
+ end
16
+ end
data/test/dsl_test.rb ADDED
@@ -0,0 +1,127 @@
1
+ require 'test_helper'
2
+
3
+ class DslTest < ActiveSupport::TestCase
4
+
5
+ test 'search' do
6
+ shop = Shop.create
7
+ seed = rand
8
+ days = [1,2,3]
9
+ opens_at = Time.now.to_i
10
+ closes_at = (Time.now + 1.day).to_i
11
+ assert_equal(
12
+ {
13
+ query: {
14
+ function_score: {
15
+ functions: [
16
+ { random_score: { seed: seed } },
17
+ {
18
+ weight: 400,
19
+ filter: {
20
+ bool: {
21
+ must: [
22
+ { term: {} }
23
+ ]
24
+ }
25
+ }
26
+ }
27
+ ]
28
+ },
29
+ query: {
30
+ filtered: {
31
+ filter: {
32
+ bool: {
33
+ should: [
34
+ { term: { :'schedules.days' => days } }
35
+ ],
36
+ must: [
37
+ { range: { :'schedules.opens_at' => { lte: opens_at } } }
38
+ ],
39
+ must_not: [
40
+ {
41
+ has_parent: {
42
+ type: 'products',
43
+ query: {
44
+ filtered: {
45
+ filter: {
46
+ bool: {
47
+ must: [
48
+ {
49
+ term: {
50
+ _parent: shop.id
51
+ }
52
+ }
53
+ ]
54
+ }
55
+ },
56
+ query: {
57
+ match_all: {}
58
+ }
59
+ }
60
+ }
61
+ }
62
+ }
63
+ ]
64
+ }
65
+ }
66
+ }
67
+ }
68
+ },
69
+ sort: [
70
+ { name: 'asc' }
71
+ ],
72
+ size: {}
73
+ },
74
+ build do
75
+ query do
76
+ function_score do
77
+ functions do
78
+ random_score do
79
+ seed seed
80
+ end
81
+ filter weight: 400 do
82
+ bool do
83
+ must do
84
+ term
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end
90
+ query do
91
+ filtered do
92
+ filter do
93
+ bool do
94
+ should do
95
+ term 'schedules.days' => days
96
+ end
97
+ must do
98
+ range 'schedules.opens_at' do
99
+ lte opens_at
100
+ end
101
+ end
102
+ must_not do
103
+ has_parent do
104
+ type 'products'
105
+ query :product, shop: shop
106
+ end
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
112
+ end
113
+ sort %w(asc) do |order|
114
+ name order
115
+ end
116
+ size
117
+ end
118
+ )
119
+ end
120
+
121
+ private
122
+
123
+ def build(&block)
124
+ Indexers::Dsl::Search.new(&block).to_h
125
+ end
126
+
127
+ end
@@ -0,0 +1,5 @@
1
+ # Add your own tasks in files placed in lib/tasks ending in .rake,
2
+ # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
3
+
4
+ require File.expand_path('../config/application', __FILE__)
5
+ Rails.application.load_tasks