elasticord 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +3 -1
- data/dash_overlord.gemspec +2 -2
- data/elasticord.gemspec +1 -1
- data/lib/dash_overlord/models/v1/dynamo_db/base.rb +22 -11
- data/lib/dash_overlord/models/v1/dynamo_db/migration.rb +13 -3
- data/lib/dash_overlord/models/v1/dynamo_db/paginated_result.rb +16 -0
- data/lib/dash_overlord/models/v1/dynamo_db/scan.rb +114 -0
- data/lib/dash_overlord/use_cases/v1/dummy_data/create/charts/base.rb +3 -2
- data/lib/dash_overlord/use_cases/v1/dummy_data/create/charts/build.rb +1 -1
- data/lib/dash_overlord/use_cases/v1/repositories/dynamo_db/create/batch.rb +3 -2
- data/lib/dash_overlord/use_cases/v1/repositories/dynamo_db/search_and_paginate/apply_filters.rb +21 -0
- data/lib/dash_overlord/use_cases/v1/repositories/dynamo_db/search_and_paginate/apply_pagination.rb +62 -0
- data/lib/dash_overlord/use_cases/v1/repositories/dynamo_db/search_and_paginate/base.rb +39 -0
- data/lib/dash_overlord/use_cases/v1/shared/search_and_paginate/apply_filters.rb +2 -2
- data/lib/dash_overlord/use_cases/v1/shared/search_and_paginate/apply_pagination.rb +4 -11
- data/lib/dash_overlord/use_cases/v1/shared/search_and_paginate/base.rb +2 -6
- data/lib/dash_overlord/use_cases/v1/shared/search_and_paginate/build_meta_data.rb +26 -0
- data/lib/dash_overlord/use_cases/v1/shared/search_and_paginate/execute_query.rb +21 -0
- data/lib/dash_overlord/use_cases/v1/videos/index/dynamodb/search_and_paginate_videos.rb +7 -2
- data/lib/dash_overlord/use_cases/v1/videos/index/elastic_search/search_and_paginate_videos.rb +2 -80
- data/lib/elasticord/base.rb +10 -11
- data/lib/elasticord/entities/array_with_meta_data.rb +37 -0
- data/lib/elasticord/orm/search_builder.rb +101 -0
- data/lib/elasticord/version.rb +1 -1
- data/lib/tasks/create_dummy_data.rake +1 -0
- data/spec/dash_overlord/use_cases/v1/videos/index/dynamo_db_spec.rb +71 -9
- data/spec/elasticord/base_spec.rb +42 -155
- data/spec/elasticord/orm/search_builder_spec.rb +269 -0
- metadata +14 -3
@@ -15,15 +15,12 @@ module DashOverlord
|
|
15
15
|
|
16
16
|
def perform
|
17
17
|
context.page = fix_page(page)
|
18
|
+
context.per_page = fix_per_page(per_page)
|
18
19
|
|
19
|
-
|
20
|
-
no_pagination
|
21
|
-
else
|
22
|
-
apply_pagination
|
23
|
-
end
|
20
|
+
page == 0 ? no_pagination : apply_pagination
|
24
21
|
end
|
25
22
|
|
26
|
-
protected
|
23
|
+
protected
|
27
24
|
|
28
25
|
def no_pagination
|
29
26
|
context.per_page = 0
|
@@ -32,13 +29,9 @@ module DashOverlord
|
|
32
29
|
end
|
33
30
|
|
34
31
|
def apply_pagination
|
35
|
-
context.per_page = fix_per_page(per_page)
|
36
|
-
|
37
32
|
context.resources = paginated_resources.page(page).per(per_page)
|
38
33
|
|
39
34
|
context.page = resources.current_page
|
40
|
-
context.total_pages = resources.total_pages
|
41
|
-
context.total_results = resources.total_count
|
42
35
|
end
|
43
36
|
|
44
37
|
def fix_page(page)
|
@@ -54,7 +47,7 @@ module DashOverlord
|
|
54
47
|
(per_page_or_nil || PER_PAGE_DEFAULT).to_i
|
55
48
|
end
|
56
49
|
|
57
|
-
private
|
50
|
+
private
|
58
51
|
|
59
52
|
def paginated_resources
|
60
53
|
if resources.respond_to?(:page)
|
@@ -18,12 +18,8 @@ module DashOverlord
|
|
18
18
|
invoke! ApplyFullTextSearch
|
19
19
|
invoke! ApplyFilters
|
20
20
|
invoke! ApplyPagination
|
21
|
-
|
22
|
-
|
23
|
-
context.attributes.slice :page,
|
24
|
-
:per_page,
|
25
|
-
:total_pages,
|
26
|
-
:total_results
|
21
|
+
invoke! ExecuteQuery
|
22
|
+
invoke! BuildMetaData
|
27
23
|
|
28
24
|
context.resources
|
29
25
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module DashOverlord
|
2
|
+
module UseCases
|
3
|
+
module V1
|
4
|
+
module Shared
|
5
|
+
module SearchAndPaginate
|
6
|
+
|
7
|
+
class BuildMetaData < V1::Base
|
8
|
+
context_reader :resources
|
9
|
+
|
10
|
+
def perform
|
11
|
+
context.total_pages ||= resources.total_pages
|
12
|
+
context.total_results ||= resources.total_count
|
13
|
+
|
14
|
+
context.meta = Entities::V1::MetaData.new \
|
15
|
+
context.attributes.slice :page,
|
16
|
+
:per_page,
|
17
|
+
:total_pages,
|
18
|
+
:total_results
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module DashOverlord
|
2
|
+
module UseCases
|
3
|
+
module V1
|
4
|
+
module Shared
|
5
|
+
module SearchAndPaginate
|
6
|
+
|
7
|
+
class ExecuteQuery < V1::Base
|
8
|
+
context_reader :resources
|
9
|
+
|
10
|
+
def perform
|
11
|
+
return unless resources.respond_to?(:load)
|
12
|
+
|
13
|
+
context.resources = resources.load
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -5,11 +5,16 @@ module DashOverlord
|
|
5
5
|
module Index
|
6
6
|
module DynamoDb
|
7
7
|
|
8
|
-
class SearchAndPaginateVideos <
|
8
|
+
class SearchAndPaginateVideos < Repositories::DynamoDb::SearchAndPaginate::Base
|
9
9
|
def perform
|
10
|
+
context.videos = search_and_paginate(resource_class)
|
10
11
|
end
|
11
12
|
|
12
|
-
protected
|
13
|
+
protected ################## PROTECTED ###################
|
14
|
+
|
15
|
+
def resource_class
|
16
|
+
DashOverlord::Models::V1::DynamoDb::Video
|
17
|
+
end
|
13
18
|
end
|
14
19
|
|
15
20
|
end
|
data/lib/dash_overlord/use_cases/v1/videos/index/elastic_search/search_and_paginate_videos.rb
CHANGED
@@ -5,87 +5,9 @@ module DashOverlord
|
|
5
5
|
module Index
|
6
6
|
module ElasticSearch
|
7
7
|
|
8
|
-
class SearchAndPaginateVideos <
|
9
|
-
PAGE_DEFAULT = 1
|
10
|
-
PER_PAGE_DEFAULT = 10
|
11
|
-
|
12
|
-
context_reader :page, :per_page, :filter_query, :full_text_term
|
13
|
-
|
8
|
+
class SearchAndPaginateVideos < Shared::SearchAndPaginate::Base
|
14
9
|
def perform
|
15
|
-
context.
|
16
|
-
context.per_page = fix_per_page(per_page)
|
17
|
-
|
18
|
-
context.videos = search_results[:data]
|
19
|
-
|
20
|
-
context.meta = Entities::V1::MetaData.new \
|
21
|
-
page: page,
|
22
|
-
per_page: per_page,
|
23
|
-
total_pages: total_pages,
|
24
|
-
total_results: total_results
|
25
|
-
end
|
26
|
-
|
27
|
-
protected
|
28
|
-
|
29
|
-
def fix_page(page)
|
30
|
-
page_or_nil = page.class == String ? page.presence : page
|
31
|
-
|
32
|
-
(page_or_nil || PAGE_DEFAULT).to_i
|
33
|
-
end
|
34
|
-
|
35
|
-
def fix_per_page(per_page)
|
36
|
-
per_page_or_nil = \
|
37
|
-
per_page.class == String ? per_page.presence : per_page
|
38
|
-
|
39
|
-
(per_page_or_nil || PER_PAGE_DEFAULT).to_i
|
40
|
-
end
|
41
|
-
|
42
|
-
def search_results
|
43
|
-
@search_results ||= Models::V1::Answer.search(search_params)
|
44
|
-
end
|
45
|
-
|
46
|
-
def total_results
|
47
|
-
search_results[:total_results]
|
48
|
-
end
|
49
|
-
|
50
|
-
def total_pages
|
51
|
-
(total_results / per_page.to_f).ceil
|
52
|
-
end
|
53
|
-
|
54
|
-
private
|
55
|
-
|
56
|
-
def search_params
|
57
|
-
search_params = {
|
58
|
-
size: per_page,
|
59
|
-
from: (page - 1) * per_page,
|
60
|
-
query: { bool: {} }
|
61
|
-
}
|
62
|
-
|
63
|
-
build_query_params(search_params)
|
64
|
-
|
65
|
-
build_filter_params(search_params)
|
66
|
-
|
67
|
-
search_params
|
68
|
-
end
|
69
|
-
|
70
|
-
def build_query_params(search_params)
|
71
|
-
return unless full_text_term
|
72
|
-
|
73
|
-
params = (search_params[:query][:bool][:must] ||= [])
|
74
|
-
|
75
|
-
# params[:must] << { match: { title: full_text_term } }
|
76
|
-
end
|
77
|
-
|
78
|
-
def build_filter_params(search_params)
|
79
|
-
return unless filter_query
|
80
|
-
|
81
|
-
params = (search_params[:query][:bool][:filter] ||= [])
|
82
|
-
|
83
|
-
filter_query.each do |key, value|
|
84
|
-
key = key.to_s.split('_eq').first.to_sym
|
85
|
-
value = value.downcase if value.is_a? String
|
86
|
-
|
87
|
-
params.push({ term: { key => value } })
|
88
|
-
end
|
10
|
+
context.videos = search_and_paginate Models::V1::Answer.none
|
89
11
|
end
|
90
12
|
end
|
91
13
|
|
data/lib/elasticord/base.rb
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
require 'ostruct'
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
3
|
+
require 'elasticord/orm/get'
|
4
|
+
require 'elasticord/orm/index'
|
5
|
+
require 'elasticord/orm/create'
|
6
|
+
require 'elasticord/orm/delete'
|
7
|
+
require 'elasticord/orm/search_builder'
|
7
8
|
|
8
9
|
module Elasticord
|
9
10
|
class Base < OpenStruct
|
@@ -26,14 +27,12 @@ module Elasticord
|
|
26
27
|
self.new(result.merge({ 'id' => id }))
|
27
28
|
end
|
28
29
|
|
29
|
-
def search(
|
30
|
-
|
31
|
-
|
32
|
-
results[:data] = results[:data].map do |attributes|
|
33
|
-
self.new(attributes)
|
34
|
-
end
|
30
|
+
def search(body = {})
|
31
|
+
none.load(body)
|
32
|
+
end
|
35
33
|
|
36
|
-
|
34
|
+
def none
|
35
|
+
ORM::SearchBuilder.new(self, index, type)
|
37
36
|
end
|
38
37
|
|
39
38
|
def delete_by_query(attributes = {})
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Elasticord
|
2
|
+
module Entities
|
3
|
+
class ArrayWithMetaData < SimpleDelegator
|
4
|
+
attr_accessor :per_page, :current_page, :total_results
|
5
|
+
|
6
|
+
def initialize(page_default, per_page_default)
|
7
|
+
@internal_array = []
|
8
|
+
|
9
|
+
@per_page = per_page_default
|
10
|
+
@current_page = page_default
|
11
|
+
@total_results = 0
|
12
|
+
|
13
|
+
super(@internal_array)
|
14
|
+
end
|
15
|
+
|
16
|
+
alias total_count total_results
|
17
|
+
|
18
|
+
def total_pages
|
19
|
+
(total_results / per_page.to_f).ceil
|
20
|
+
end
|
21
|
+
|
22
|
+
def size
|
23
|
+
per_page
|
24
|
+
end
|
25
|
+
|
26
|
+
def from
|
27
|
+
(current_page - 1) * size
|
28
|
+
end
|
29
|
+
|
30
|
+
protected
|
31
|
+
|
32
|
+
def parse_string_number(number)
|
33
|
+
number.class == String ? number.presence : number
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
require 'elasticord/entities/array_with_meta_data'
|
3
|
+
|
4
|
+
module Elasticord
|
5
|
+
module ORM
|
6
|
+
class SearchBuilder
|
7
|
+
extend Forwardable
|
8
|
+
|
9
|
+
PAGE_DEFAULT = 1
|
10
|
+
PER_PAGE_DEFAULT = 10
|
11
|
+
|
12
|
+
def initialize(entity, index, type)
|
13
|
+
@entity, @index, @type = entity, index, type
|
14
|
+
end
|
15
|
+
|
16
|
+
def_delegators :results, :current_page, :per_page, :size, :from
|
17
|
+
|
18
|
+
def page(new_page)
|
19
|
+
results.current_page = new_page
|
20
|
+
|
21
|
+
body_params[:from] = results.from
|
22
|
+
|
23
|
+
self
|
24
|
+
end
|
25
|
+
|
26
|
+
def per(new_per_page)
|
27
|
+
results.per_page = new_per_page
|
28
|
+
|
29
|
+
body_params[:size] = results.size
|
30
|
+
body_params[:from] = results.from
|
31
|
+
|
32
|
+
self
|
33
|
+
end
|
34
|
+
|
35
|
+
def load(body = {})
|
36
|
+
begin
|
37
|
+
elastic_results = execute_search(body)
|
38
|
+
|
39
|
+
results.total_results = elastic_results['total']
|
40
|
+
|
41
|
+
elastic_results['hits'].map do |result|
|
42
|
+
results.push entity.new(result['_source'])
|
43
|
+
end
|
44
|
+
rescue Elasticsearch::Transport::Transport::Errors::NotFound
|
45
|
+
nil
|
46
|
+
end
|
47
|
+
|
48
|
+
results
|
49
|
+
end
|
50
|
+
|
51
|
+
# def ft(full_text_term)
|
52
|
+
# return self unless full_text_term
|
53
|
+
#
|
54
|
+
# params = (body_params[:query][:bool][:must] ||= [])
|
55
|
+
#
|
56
|
+
# # params[:must] << { match: { title: full_text_term } }
|
57
|
+
# end
|
58
|
+
|
59
|
+
def ransack(ransack_filters = {})
|
60
|
+
return self unless ransack_filters
|
61
|
+
|
62
|
+
params = (body_params[:query][:bool][:filter] ||= [])
|
63
|
+
|
64
|
+
ransack_filters.each do |key, value|
|
65
|
+
key = key.to_s.split('_eq').first.to_sym
|
66
|
+
value = value.downcase if value.is_a? String
|
67
|
+
|
68
|
+
params.push({ term: { key => value } })
|
69
|
+
end
|
70
|
+
|
71
|
+
self
|
72
|
+
end
|
73
|
+
|
74
|
+
def result(*_args)
|
75
|
+
self
|
76
|
+
end
|
77
|
+
|
78
|
+
def body_params
|
79
|
+
@body_params ||= { size: size, from: from, query: { bool: {} } }
|
80
|
+
end
|
81
|
+
|
82
|
+
protected
|
83
|
+
|
84
|
+
attr_reader :type, :index, :entity
|
85
|
+
|
86
|
+
def results
|
87
|
+
@results ||= Entities::ArrayWithMetaData.new \
|
88
|
+
PAGE_DEFAULT, PER_PAGE_DEFAULT
|
89
|
+
end
|
90
|
+
|
91
|
+
def execute_search(body)
|
92
|
+
elastic_results = Elasticord.client.search \
|
93
|
+
index: index, type: type, body: body_params.merge(body)
|
94
|
+
|
95
|
+
hits = elastic_results['hits']
|
96
|
+
|
97
|
+
(hits && hits['total'].to_i > 0) ? hits : { 'hits' => [], 'total' => 0 }
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
data/lib/elasticord/version.rb
CHANGED
@@ -1,11 +1,18 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe DashOverlord::UseCases::V1::Videos::Index::
|
3
|
+
describe DashOverlord::UseCases::V1::Videos::Index::Base,
|
4
4
|
dynamodb: true do
|
5
5
|
|
6
|
+
def search_adapter
|
7
|
+
DashOverlord::UseCases::V1::Videos::Index::DynamoDb::Base
|
8
|
+
end
|
9
|
+
|
6
10
|
def make_the_call(params = nil)
|
7
11
|
@context = described_class.perform \
|
8
|
-
(params || {}).merge({
|
12
|
+
(params || {}).merge({
|
13
|
+
search_adapter_use_case: search_adapter,
|
14
|
+
session: user_session
|
15
|
+
})
|
9
16
|
|
10
17
|
raise_context_unknown_error @context
|
11
18
|
end
|
@@ -15,6 +22,7 @@ describe DashOverlord::UseCases::V1::Videos::Index::DynamoDb::Base,
|
|
15
22
|
DashOverlord::Models::V1::DynamoDb::Video
|
16
23
|
|
17
24
|
migration.create!
|
25
|
+
|
18
26
|
end
|
19
27
|
|
20
28
|
after(:all) do
|
@@ -29,14 +37,14 @@ describe DashOverlord::UseCases::V1::Videos::Index::DynamoDb::Base,
|
|
29
37
|
"url": "https://s3-eu-west-1.amazonaws.com/streetbees-assets/dashboards/home_selfie/videos_1/11264-kitchen.MOV",
|
30
38
|
"original_url": "https://streetbees-photos.s3-eu-west-1.amazonaws.com/uploads/bee_videos/temporary/original_11264-kitchen.jpg",
|
31
39
|
"subtitle_url": "/subtitles//unilever/hhc/11264-kitchen.en.vtt",
|
32
|
-
"country": "UK",
|
40
|
+
"country": ["UK", "US", "PT", "GB"].sample,
|
33
41
|
"category": "Kitchen floor",
|
34
42
|
"extra": nil,
|
35
43
|
"segment_uid": "hhc",
|
36
44
|
"lsm": "13+",
|
37
45
|
"age": nil,
|
38
46
|
"room": "Kitchen",
|
39
|
-
"gender": "Male",
|
47
|
+
"gender": ["Male", "Female"].sample,
|
40
48
|
"user_id": 835,
|
41
49
|
"children": true,
|
42
50
|
"age_group": "46+",
|
@@ -46,16 +54,21 @@ describe DashOverlord::UseCases::V1::Videos::Index::DynamoDb::Base,
|
|
46
54
|
end
|
47
55
|
|
48
56
|
def create_answers
|
49
|
-
videos = 20.times.map do |index|
|
50
|
-
build_answer.merge(
|
57
|
+
@videos = 20.times.map do |index|
|
58
|
+
build_answer.merge(
|
59
|
+
"id" => (index + 1),
|
60
|
+
"dashboard_id": 1
|
61
|
+
)
|
51
62
|
end
|
52
63
|
|
53
64
|
DashOverlord::Models::V1::DynamoDb::Video.create_all \
|
54
|
-
videos
|
65
|
+
@videos
|
55
66
|
end
|
56
67
|
|
57
68
|
context 'when no filter params are used' do
|
58
69
|
before do
|
70
|
+
@current_user = create('v1/user', :unilever)
|
71
|
+
|
59
72
|
create_answers
|
60
73
|
|
61
74
|
make_the_call
|
@@ -73,12 +86,61 @@ describe DashOverlord::UseCases::V1::Videos::Index::DynamoDb::Base,
|
|
73
86
|
it 'should return pagination meta_data' do
|
74
87
|
expect(@context.meta.page).to be 1
|
75
88
|
expect(@context.meta.per_page).to be 10
|
76
|
-
expect(@context.meta.total_pages).to be 2
|
77
|
-
expect(@context.meta.total_results).to be 20
|
78
89
|
end
|
79
90
|
|
80
91
|
it 'should return 10 results' do
|
81
92
|
expect(@context.videos.length).to be 10
|
82
93
|
end
|
83
94
|
end
|
95
|
+
|
96
|
+
context 'when filtering by page = 0' do
|
97
|
+
before do
|
98
|
+
@current_user = create('v1/user', :unilever)
|
99
|
+
|
100
|
+
create_answers
|
101
|
+
|
102
|
+
make_the_call page: 0
|
103
|
+
end
|
104
|
+
|
105
|
+
it '@context.status should be ok' do
|
106
|
+
expect(@context.status).to eq 'ok'
|
107
|
+
end
|
108
|
+
|
109
|
+
it 'should return pagination meta_data' do
|
110
|
+
expect(@context.meta.page).to be 0
|
111
|
+
expect(@context.meta.per_page).to be 0
|
112
|
+
end
|
113
|
+
|
114
|
+
it 'should return 20 results', :cana do
|
115
|
+
expect(@context.videos.length).to be 20
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
context 'when filtering by something' do
|
120
|
+
before do
|
121
|
+
@current_user = create('v1/user', :unilever)
|
122
|
+
|
123
|
+
create_answers
|
124
|
+
|
125
|
+
@female_videos = @videos.select{|el| el[:gender] == "Female" }
|
126
|
+
|
127
|
+
make_the_call \
|
128
|
+
filter_query: {
|
129
|
+
gender_eq: 'Female'
|
130
|
+
}
|
131
|
+
end
|
132
|
+
|
133
|
+
it '@context.status should be ok' do
|
134
|
+
expect(@context.status).to eq 'ok'
|
135
|
+
end
|
136
|
+
|
137
|
+
it 'should return pagination meta_data' do
|
138
|
+
expect(@context.meta.page).to be 1
|
139
|
+
expect(@context.meta.per_page).to be 10
|
140
|
+
end
|
141
|
+
|
142
|
+
it 'should return only female results', :focusrite do
|
143
|
+
expect(@context.videos.length).to be @female_videos.count
|
144
|
+
end
|
145
|
+
end
|
84
146
|
end
|