elasticord 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +3 -1
- data/dash_overlord.gemspec +2 -2
- data/elasticord.gemspec +1 -1
- data/lib/dash_overlord/models/v1/dynamo_db/base.rb +22 -11
- data/lib/dash_overlord/models/v1/dynamo_db/migration.rb +13 -3
- data/lib/dash_overlord/models/v1/dynamo_db/paginated_result.rb +16 -0
- data/lib/dash_overlord/models/v1/dynamo_db/scan.rb +114 -0
- data/lib/dash_overlord/use_cases/v1/dummy_data/create/charts/base.rb +3 -2
- data/lib/dash_overlord/use_cases/v1/dummy_data/create/charts/build.rb +1 -1
- data/lib/dash_overlord/use_cases/v1/repositories/dynamo_db/create/batch.rb +3 -2
- data/lib/dash_overlord/use_cases/v1/repositories/dynamo_db/search_and_paginate/apply_filters.rb +21 -0
- data/lib/dash_overlord/use_cases/v1/repositories/dynamo_db/search_and_paginate/apply_pagination.rb +62 -0
- data/lib/dash_overlord/use_cases/v1/repositories/dynamo_db/search_and_paginate/base.rb +39 -0
- data/lib/dash_overlord/use_cases/v1/shared/search_and_paginate/apply_filters.rb +2 -2
- data/lib/dash_overlord/use_cases/v1/shared/search_and_paginate/apply_pagination.rb +4 -11
- data/lib/dash_overlord/use_cases/v1/shared/search_and_paginate/base.rb +2 -6
- data/lib/dash_overlord/use_cases/v1/shared/search_and_paginate/build_meta_data.rb +26 -0
- data/lib/dash_overlord/use_cases/v1/shared/search_and_paginate/execute_query.rb +21 -0
- data/lib/dash_overlord/use_cases/v1/videos/index/dynamodb/search_and_paginate_videos.rb +7 -2
- data/lib/dash_overlord/use_cases/v1/videos/index/elastic_search/search_and_paginate_videos.rb +2 -80
- data/lib/elasticord/base.rb +10 -11
- data/lib/elasticord/entities/array_with_meta_data.rb +37 -0
- data/lib/elasticord/orm/search_builder.rb +101 -0
- data/lib/elasticord/version.rb +1 -1
- data/lib/tasks/create_dummy_data.rake +1 -0
- data/spec/dash_overlord/use_cases/v1/videos/index/dynamo_db_spec.rb +71 -9
- data/spec/elasticord/base_spec.rb +42 -155
- data/spec/elasticord/orm/search_builder_spec.rb +269 -0
- metadata +14 -3
@@ -15,15 +15,12 @@ module DashOverlord
|
|
15
15
|
|
16
16
|
def perform
|
17
17
|
context.page = fix_page(page)
|
18
|
+
context.per_page = fix_per_page(per_page)
|
18
19
|
|
19
|
-
|
20
|
-
no_pagination
|
21
|
-
else
|
22
|
-
apply_pagination
|
23
|
-
end
|
20
|
+
page == 0 ? no_pagination : apply_pagination
|
24
21
|
end
|
25
22
|
|
26
|
-
protected
|
23
|
+
protected
|
27
24
|
|
28
25
|
def no_pagination
|
29
26
|
context.per_page = 0
|
@@ -32,13 +29,9 @@ module DashOverlord
|
|
32
29
|
end
|
33
30
|
|
34
31
|
def apply_pagination
|
35
|
-
context.per_page = fix_per_page(per_page)
|
36
|
-
|
37
32
|
context.resources = paginated_resources.page(page).per(per_page)
|
38
33
|
|
39
34
|
context.page = resources.current_page
|
40
|
-
context.total_pages = resources.total_pages
|
41
|
-
context.total_results = resources.total_count
|
42
35
|
end
|
43
36
|
|
44
37
|
def fix_page(page)
|
@@ -54,7 +47,7 @@ module DashOverlord
|
|
54
47
|
(per_page_or_nil || PER_PAGE_DEFAULT).to_i
|
55
48
|
end
|
56
49
|
|
57
|
-
private
|
50
|
+
private
|
58
51
|
|
59
52
|
def paginated_resources
|
60
53
|
if resources.respond_to?(:page)
|
@@ -18,12 +18,8 @@ module DashOverlord
|
|
18
18
|
invoke! ApplyFullTextSearch
|
19
19
|
invoke! ApplyFilters
|
20
20
|
invoke! ApplyPagination
|
21
|
-
|
22
|
-
|
23
|
-
context.attributes.slice :page,
|
24
|
-
:per_page,
|
25
|
-
:total_pages,
|
26
|
-
:total_results
|
21
|
+
invoke! ExecuteQuery
|
22
|
+
invoke! BuildMetaData
|
27
23
|
|
28
24
|
context.resources
|
29
25
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module DashOverlord
|
2
|
+
module UseCases
|
3
|
+
module V1
|
4
|
+
module Shared
|
5
|
+
module SearchAndPaginate
|
6
|
+
|
7
|
+
class BuildMetaData < V1::Base
|
8
|
+
context_reader :resources
|
9
|
+
|
10
|
+
def perform
|
11
|
+
context.total_pages ||= resources.total_pages
|
12
|
+
context.total_results ||= resources.total_count
|
13
|
+
|
14
|
+
context.meta = Entities::V1::MetaData.new \
|
15
|
+
context.attributes.slice :page,
|
16
|
+
:per_page,
|
17
|
+
:total_pages,
|
18
|
+
:total_results
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module DashOverlord
|
2
|
+
module UseCases
|
3
|
+
module V1
|
4
|
+
module Shared
|
5
|
+
module SearchAndPaginate
|
6
|
+
|
7
|
+
class ExecuteQuery < V1::Base
|
8
|
+
context_reader :resources
|
9
|
+
|
10
|
+
def perform
|
11
|
+
return unless resources.respond_to?(:load)
|
12
|
+
|
13
|
+
context.resources = resources.load
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -5,11 +5,16 @@ module DashOverlord
|
|
5
5
|
module Index
|
6
6
|
module DynamoDb
|
7
7
|
|
8
|
-
class SearchAndPaginateVideos <
|
8
|
+
class SearchAndPaginateVideos < Repositories::DynamoDb::SearchAndPaginate::Base
|
9
9
|
def perform
|
10
|
+
context.videos = search_and_paginate(resource_class)
|
10
11
|
end
|
11
12
|
|
12
|
-
protected
|
13
|
+
protected ################## PROTECTED ###################
|
14
|
+
|
15
|
+
def resource_class
|
16
|
+
DashOverlord::Models::V1::DynamoDb::Video
|
17
|
+
end
|
13
18
|
end
|
14
19
|
|
15
20
|
end
|
data/lib/dash_overlord/use_cases/v1/videos/index/elastic_search/search_and_paginate_videos.rb
CHANGED
@@ -5,87 +5,9 @@ module DashOverlord
|
|
5
5
|
module Index
|
6
6
|
module ElasticSearch
|
7
7
|
|
8
|
-
class SearchAndPaginateVideos <
|
9
|
-
PAGE_DEFAULT = 1
|
10
|
-
PER_PAGE_DEFAULT = 10
|
11
|
-
|
12
|
-
context_reader :page, :per_page, :filter_query, :full_text_term
|
13
|
-
|
8
|
+
class SearchAndPaginateVideos < Shared::SearchAndPaginate::Base
|
14
9
|
def perform
|
15
|
-
context.
|
16
|
-
context.per_page = fix_per_page(per_page)
|
17
|
-
|
18
|
-
context.videos = search_results[:data]
|
19
|
-
|
20
|
-
context.meta = Entities::V1::MetaData.new \
|
21
|
-
page: page,
|
22
|
-
per_page: per_page,
|
23
|
-
total_pages: total_pages,
|
24
|
-
total_results: total_results
|
25
|
-
end
|
26
|
-
|
27
|
-
protected
|
28
|
-
|
29
|
-
def fix_page(page)
|
30
|
-
page_or_nil = page.class == String ? page.presence : page
|
31
|
-
|
32
|
-
(page_or_nil || PAGE_DEFAULT).to_i
|
33
|
-
end
|
34
|
-
|
35
|
-
def fix_per_page(per_page)
|
36
|
-
per_page_or_nil = \
|
37
|
-
per_page.class == String ? per_page.presence : per_page
|
38
|
-
|
39
|
-
(per_page_or_nil || PER_PAGE_DEFAULT).to_i
|
40
|
-
end
|
41
|
-
|
42
|
-
def search_results
|
43
|
-
@search_results ||= Models::V1::Answer.search(search_params)
|
44
|
-
end
|
45
|
-
|
46
|
-
def total_results
|
47
|
-
search_results[:total_results]
|
48
|
-
end
|
49
|
-
|
50
|
-
def total_pages
|
51
|
-
(total_results / per_page.to_f).ceil
|
52
|
-
end
|
53
|
-
|
54
|
-
private
|
55
|
-
|
56
|
-
def search_params
|
57
|
-
search_params = {
|
58
|
-
size: per_page,
|
59
|
-
from: (page - 1) * per_page,
|
60
|
-
query: { bool: {} }
|
61
|
-
}
|
62
|
-
|
63
|
-
build_query_params(search_params)
|
64
|
-
|
65
|
-
build_filter_params(search_params)
|
66
|
-
|
67
|
-
search_params
|
68
|
-
end
|
69
|
-
|
70
|
-
def build_query_params(search_params)
|
71
|
-
return unless full_text_term
|
72
|
-
|
73
|
-
params = (search_params[:query][:bool][:must] ||= [])
|
74
|
-
|
75
|
-
# params[:must] << { match: { title: full_text_term } }
|
76
|
-
end
|
77
|
-
|
78
|
-
def build_filter_params(search_params)
|
79
|
-
return unless filter_query
|
80
|
-
|
81
|
-
params = (search_params[:query][:bool][:filter] ||= [])
|
82
|
-
|
83
|
-
filter_query.each do |key, value|
|
84
|
-
key = key.to_s.split('_eq').first.to_sym
|
85
|
-
value = value.downcase if value.is_a? String
|
86
|
-
|
87
|
-
params.push({ term: { key => value } })
|
88
|
-
end
|
10
|
+
context.videos = search_and_paginate Models::V1::Answer.none
|
89
11
|
end
|
90
12
|
end
|
91
13
|
|
data/lib/elasticord/base.rb
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
require 'ostruct'
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
3
|
+
require 'elasticord/orm/get'
|
4
|
+
require 'elasticord/orm/index'
|
5
|
+
require 'elasticord/orm/create'
|
6
|
+
require 'elasticord/orm/delete'
|
7
|
+
require 'elasticord/orm/search_builder'
|
7
8
|
|
8
9
|
module Elasticord
|
9
10
|
class Base < OpenStruct
|
@@ -26,14 +27,12 @@ module Elasticord
|
|
26
27
|
self.new(result.merge({ 'id' => id }))
|
27
28
|
end
|
28
29
|
|
29
|
-
def search(
|
30
|
-
|
31
|
-
|
32
|
-
results[:data] = results[:data].map do |attributes|
|
33
|
-
self.new(attributes)
|
34
|
-
end
|
30
|
+
def search(body = {})
|
31
|
+
none.load(body)
|
32
|
+
end
|
35
33
|
|
36
|
-
|
34
|
+
def none
|
35
|
+
ORM::SearchBuilder.new(self, index, type)
|
37
36
|
end
|
38
37
|
|
39
38
|
def delete_by_query(attributes = {})
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Elasticord
|
2
|
+
module Entities
|
3
|
+
class ArrayWithMetaData < SimpleDelegator
|
4
|
+
attr_accessor :per_page, :current_page, :total_results
|
5
|
+
|
6
|
+
def initialize(page_default, per_page_default)
|
7
|
+
@internal_array = []
|
8
|
+
|
9
|
+
@per_page = per_page_default
|
10
|
+
@current_page = page_default
|
11
|
+
@total_results = 0
|
12
|
+
|
13
|
+
super(@internal_array)
|
14
|
+
end
|
15
|
+
|
16
|
+
alias total_count total_results
|
17
|
+
|
18
|
+
def total_pages
|
19
|
+
(total_results / per_page.to_f).ceil
|
20
|
+
end
|
21
|
+
|
22
|
+
def size
|
23
|
+
per_page
|
24
|
+
end
|
25
|
+
|
26
|
+
def from
|
27
|
+
(current_page - 1) * size
|
28
|
+
end
|
29
|
+
|
30
|
+
protected
|
31
|
+
|
32
|
+
def parse_string_number(number)
|
33
|
+
number.class == String ? number.presence : number
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
require 'elasticord/entities/array_with_meta_data'
|
3
|
+
|
4
|
+
module Elasticord
|
5
|
+
module ORM
|
6
|
+
class SearchBuilder
|
7
|
+
extend Forwardable
|
8
|
+
|
9
|
+
PAGE_DEFAULT = 1
|
10
|
+
PER_PAGE_DEFAULT = 10
|
11
|
+
|
12
|
+
def initialize(entity, index, type)
|
13
|
+
@entity, @index, @type = entity, index, type
|
14
|
+
end
|
15
|
+
|
16
|
+
def_delegators :results, :current_page, :per_page, :size, :from
|
17
|
+
|
18
|
+
def page(new_page)
|
19
|
+
results.current_page = new_page
|
20
|
+
|
21
|
+
body_params[:from] = results.from
|
22
|
+
|
23
|
+
self
|
24
|
+
end
|
25
|
+
|
26
|
+
def per(new_per_page)
|
27
|
+
results.per_page = new_per_page
|
28
|
+
|
29
|
+
body_params[:size] = results.size
|
30
|
+
body_params[:from] = results.from
|
31
|
+
|
32
|
+
self
|
33
|
+
end
|
34
|
+
|
35
|
+
def load(body = {})
|
36
|
+
begin
|
37
|
+
elastic_results = execute_search(body)
|
38
|
+
|
39
|
+
results.total_results = elastic_results['total']
|
40
|
+
|
41
|
+
elastic_results['hits'].map do |result|
|
42
|
+
results.push entity.new(result['_source'])
|
43
|
+
end
|
44
|
+
rescue Elasticsearch::Transport::Transport::Errors::NotFound
|
45
|
+
nil
|
46
|
+
end
|
47
|
+
|
48
|
+
results
|
49
|
+
end
|
50
|
+
|
51
|
+
# def ft(full_text_term)
|
52
|
+
# return self unless full_text_term
|
53
|
+
#
|
54
|
+
# params = (body_params[:query][:bool][:must] ||= [])
|
55
|
+
#
|
56
|
+
# # params[:must] << { match: { title: full_text_term } }
|
57
|
+
# end
|
58
|
+
|
59
|
+
def ransack(ransack_filters = {})
|
60
|
+
return self unless ransack_filters
|
61
|
+
|
62
|
+
params = (body_params[:query][:bool][:filter] ||= [])
|
63
|
+
|
64
|
+
ransack_filters.each do |key, value|
|
65
|
+
key = key.to_s.split('_eq').first.to_sym
|
66
|
+
value = value.downcase if value.is_a? String
|
67
|
+
|
68
|
+
params.push({ term: { key => value } })
|
69
|
+
end
|
70
|
+
|
71
|
+
self
|
72
|
+
end
|
73
|
+
|
74
|
+
def result(*_args)
|
75
|
+
self
|
76
|
+
end
|
77
|
+
|
78
|
+
def body_params
|
79
|
+
@body_params ||= { size: size, from: from, query: { bool: {} } }
|
80
|
+
end
|
81
|
+
|
82
|
+
protected
|
83
|
+
|
84
|
+
attr_reader :type, :index, :entity
|
85
|
+
|
86
|
+
def results
|
87
|
+
@results ||= Entities::ArrayWithMetaData.new \
|
88
|
+
PAGE_DEFAULT, PER_PAGE_DEFAULT
|
89
|
+
end
|
90
|
+
|
91
|
+
def execute_search(body)
|
92
|
+
elastic_results = Elasticord.client.search \
|
93
|
+
index: index, type: type, body: body_params.merge(body)
|
94
|
+
|
95
|
+
hits = elastic_results['hits']
|
96
|
+
|
97
|
+
(hits && hits['total'].to_i > 0) ? hits : { 'hits' => [], 'total' => 0 }
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
data/lib/elasticord/version.rb
CHANGED
@@ -1,11 +1,18 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe DashOverlord::UseCases::V1::Videos::Index::
|
3
|
+
describe DashOverlord::UseCases::V1::Videos::Index::Base,
|
4
4
|
dynamodb: true do
|
5
5
|
|
6
|
+
def search_adapter
|
7
|
+
DashOverlord::UseCases::V1::Videos::Index::DynamoDb::Base
|
8
|
+
end
|
9
|
+
|
6
10
|
def make_the_call(params = nil)
|
7
11
|
@context = described_class.perform \
|
8
|
-
(params || {}).merge({
|
12
|
+
(params || {}).merge({
|
13
|
+
search_adapter_use_case: search_adapter,
|
14
|
+
session: user_session
|
15
|
+
})
|
9
16
|
|
10
17
|
raise_context_unknown_error @context
|
11
18
|
end
|
@@ -15,6 +22,7 @@ describe DashOverlord::UseCases::V1::Videos::Index::DynamoDb::Base,
|
|
15
22
|
DashOverlord::Models::V1::DynamoDb::Video
|
16
23
|
|
17
24
|
migration.create!
|
25
|
+
|
18
26
|
end
|
19
27
|
|
20
28
|
after(:all) do
|
@@ -29,14 +37,14 @@ describe DashOverlord::UseCases::V1::Videos::Index::DynamoDb::Base,
|
|
29
37
|
"url": "https://s3-eu-west-1.amazonaws.com/streetbees-assets/dashboards/home_selfie/videos_1/11264-kitchen.MOV",
|
30
38
|
"original_url": "https://streetbees-photos.s3-eu-west-1.amazonaws.com/uploads/bee_videos/temporary/original_11264-kitchen.jpg",
|
31
39
|
"subtitle_url": "/subtitles//unilever/hhc/11264-kitchen.en.vtt",
|
32
|
-
"country": "UK",
|
40
|
+
"country": ["UK", "US", "PT", "GB"].sample,
|
33
41
|
"category": "Kitchen floor",
|
34
42
|
"extra": nil,
|
35
43
|
"segment_uid": "hhc",
|
36
44
|
"lsm": "13+",
|
37
45
|
"age": nil,
|
38
46
|
"room": "Kitchen",
|
39
|
-
"gender": "Male",
|
47
|
+
"gender": ["Male", "Female"].sample,
|
40
48
|
"user_id": 835,
|
41
49
|
"children": true,
|
42
50
|
"age_group": "46+",
|
@@ -46,16 +54,21 @@ describe DashOverlord::UseCases::V1::Videos::Index::DynamoDb::Base,
|
|
46
54
|
end
|
47
55
|
|
48
56
|
def create_answers
|
49
|
-
videos = 20.times.map do |index|
|
50
|
-
build_answer.merge(
|
57
|
+
@videos = 20.times.map do |index|
|
58
|
+
build_answer.merge(
|
59
|
+
"id" => (index + 1),
|
60
|
+
"dashboard_id": 1
|
61
|
+
)
|
51
62
|
end
|
52
63
|
|
53
64
|
DashOverlord::Models::V1::DynamoDb::Video.create_all \
|
54
|
-
videos
|
65
|
+
@videos
|
55
66
|
end
|
56
67
|
|
57
68
|
context 'when no filter params are used' do
|
58
69
|
before do
|
70
|
+
@current_user = create('v1/user', :unilever)
|
71
|
+
|
59
72
|
create_answers
|
60
73
|
|
61
74
|
make_the_call
|
@@ -73,12 +86,61 @@ describe DashOverlord::UseCases::V1::Videos::Index::DynamoDb::Base,
|
|
73
86
|
it 'should return pagination meta_data' do
|
74
87
|
expect(@context.meta.page).to be 1
|
75
88
|
expect(@context.meta.per_page).to be 10
|
76
|
-
expect(@context.meta.total_pages).to be 2
|
77
|
-
expect(@context.meta.total_results).to be 20
|
78
89
|
end
|
79
90
|
|
80
91
|
it 'should return 10 results' do
|
81
92
|
expect(@context.videos.length).to be 10
|
82
93
|
end
|
83
94
|
end
|
95
|
+
|
96
|
+
context 'when filtering by page = 0' do
|
97
|
+
before do
|
98
|
+
@current_user = create('v1/user', :unilever)
|
99
|
+
|
100
|
+
create_answers
|
101
|
+
|
102
|
+
make_the_call page: 0
|
103
|
+
end
|
104
|
+
|
105
|
+
it '@context.status should be ok' do
|
106
|
+
expect(@context.status).to eq 'ok'
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'should return pagination meta_data' do
|
110
|
+
expect(@context.meta.page).to be 0
|
111
|
+
expect(@context.meta.per_page).to be 0
|
112
|
+
end
|
113
|
+
|
114
|
+
it 'should return 20 results', :cana do
|
115
|
+
expect(@context.videos.length).to be 20
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
context 'when filtering by something' do
|
120
|
+
before do
|
121
|
+
@current_user = create('v1/user', :unilever)
|
122
|
+
|
123
|
+
create_answers
|
124
|
+
|
125
|
+
@female_videos = @videos.select{|el| el[:gender] == "Female" }
|
126
|
+
|
127
|
+
make_the_call \
|
128
|
+
filter_query: {
|
129
|
+
gender_eq: 'Female'
|
130
|
+
}
|
131
|
+
end
|
132
|
+
|
133
|
+
it '@context.status should be ok' do
|
134
|
+
expect(@context.status).to eq 'ok'
|
135
|
+
end
|
136
|
+
|
137
|
+
it 'should return pagination meta_data' do
|
138
|
+
expect(@context.meta.page).to be 1
|
139
|
+
expect(@context.meta.per_page).to be 10
|
140
|
+
end
|
141
|
+
|
142
|
+
it 'should return only female results', :focusrite do
|
143
|
+
expect(@context.videos.length).to be @female_videos.count
|
144
|
+
end
|
145
|
+
end
|
84
146
|
end
|