redis_counters-dumpers 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/Gemfile +4 -0
- data/lib/redis_counters/dumpers/destination.rb +67 -18
- data/lib/redis_counters/dumpers/dsl/destination.rb +4 -0
- data/lib/redis_counters/dumpers/engine.rb +16 -19
- data/lib/redis_counters/dumpers/version.rb +1 -1
- data/redis_counters-dumpers.gemspec +0 -1
- data/spec/internal/app/models/stat.rb +3 -0
- data/spec/internal/app/models/stats_by_day.rb +1 -0
- data/spec/internal/app/models/stats_total.rb +1 -0
- data/spec/internal/db/schema.rb +24 -0
- data/spec/lib/redis_counters/dumpers/engine_spec.rb +155 -34
- data/spec/spec_helper.rb +5 -0
- metadata +21 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: baf28efb0b4f140dae575fd7aa300a1dbb1486ec
|
4
|
+
data.tar.gz: e525a5a8049f12af5c7985a77dc6748901b64181
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d808c8e16e3438efc682c06161a51d05612fb02c3fbb286f1cd68fa714eedc25d4be52b76b0b4cae3a579b36c07726abd9946f02f11195ae18958e6535a2c4ae
|
7
|
+
data.tar.gz: d51ac56268477487b776b17fa583b5a5ab020af09acd54ae445d42f3ad4f8c94d34917da02585b2b870f8c0a3c49343579b1de69fc393ab21513d96079acde05
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
|
2
|
+
#### [Current]
|
3
|
+
* 2015-09-10 [1f96c39](../../commit/1f96c39) - __(Zhidkov Denis)__ feat: add source_conditions method
|
4
|
+
* 2015-02-19 [8885300](../../commit/8885300) - __(bibendi)__ add group_by method
|
5
|
+
|
6
|
+
#### v0.0.1
|
7
|
+
* 2015-01-26 [86918bd](../../commit/86918bd) - __(bibendi)__ initial code base
|
8
|
+
* 2015-01-26 [a238032](../../commit/a238032) - __(bibendi)__ initial
|
data/Gemfile
CHANGED
@@ -4,6 +4,10 @@ source 'https://rubygems.org'
|
|
4
4
|
|
5
5
|
group :development, :test do
|
6
6
|
gem 'combustion', github: 'pat/combustion', ref: '7d0d24c3f36ce0eb336177fc493be0721bc26665'
|
7
|
+
gem 'activerecord-postgres-hstore', require: false
|
8
|
+
gem 'simple_hstore_accessor', '~> 0.2', require: false
|
7
9
|
end
|
8
10
|
|
11
|
+
gem 'rack', '< 2' if RUBY_VERSION < '2.2.0'
|
12
|
+
|
9
13
|
gemspec
|
@@ -58,53 +58,96 @@ module RedisCounters
|
|
58
58
|
# Условия соеденяются через AND.
|
59
59
|
attr_accessor :conditions
|
60
60
|
|
61
|
+
# Список дополнительных условий, которые применяются для выборки из source-таблицы для обновления
|
62
|
+
# target, Array of String.
|
63
|
+
# Каждое условие представляет собой строку - часть SQL выражения, которое может включать именованные
|
64
|
+
# параметры из числа доступных в хеше общих параметров дампера: engine.common_params.
|
65
|
+
# Условия соединяются через AND.
|
66
|
+
attr_accessor :source_conditions
|
67
|
+
|
61
68
|
def initialize(engine)
|
62
69
|
@engine = engine
|
63
70
|
@fields_map = HashWithIndifferentAccess.new
|
64
71
|
@conditions = []
|
72
|
+
@source_conditions = []
|
65
73
|
end
|
66
74
|
|
67
75
|
def merge
|
76
|
+
sql = generate_query
|
77
|
+
sql = model.send(:sanitize_sql, [sql, engine.common_params])
|
78
|
+
connection.execute sql
|
79
|
+
end
|
80
|
+
|
81
|
+
def_delegator :model, :connection
|
82
|
+
def_delegator :model, :quoted_table_name, :target_table
|
83
|
+
def_delegator :engine, :temp_table_name, :source_table
|
84
|
+
|
85
|
+
protected
|
86
|
+
|
87
|
+
def generate_query
|
68
88
|
target_fields = fields.join(', ')
|
89
|
+
temp_source = "_source_#{source_table}"
|
90
|
+
|
91
|
+
query = create_temp_table_query(temp_source)
|
92
|
+
|
93
|
+
if increment_fields.present?
|
94
|
+
query.concat(insert_with_update_query(temp_source, target_fields))
|
95
|
+
else
|
96
|
+
query.concat(insert_without_update_query(temp_source, target_fields))
|
97
|
+
end
|
98
|
+
|
99
|
+
query.concat(drop_temp_table_query(temp_source))
|
100
|
+
query
|
101
|
+
end
|
102
|
+
|
103
|
+
def create_temp_table_query(temp_source)
|
104
|
+
<<-SQL
|
105
|
+
CREATE TEMP TABLE #{temp_source} ON COMMIT DROP AS
|
106
|
+
SELECT #{selected_fields_expression}
|
107
|
+
FROM #{source_table}
|
108
|
+
#{source_conditions_expression}
|
109
|
+
#{group_by_expression};
|
110
|
+
SQL
|
111
|
+
end
|
69
112
|
|
70
|
-
|
113
|
+
def drop_temp_table_query(temp_source)
|
114
|
+
<<-SQL
|
115
|
+
DROP TABLE #{temp_source};
|
116
|
+
SQL
|
117
|
+
end
|
118
|
+
|
119
|
+
def insert_with_update_query(temp_source, target_fields)
|
120
|
+
<<-SQL
|
71
121
|
WITH
|
72
|
-
source AS
|
73
|
-
(
|
74
|
-
SELECT #{selected_fields_expression}
|
75
|
-
FROM #{source_table}
|
76
|
-
#{group_by_expression}
|
77
|
-
),
|
78
122
|
updated AS
|
79
123
|
(
|
80
124
|
UPDATE #{target_table} target
|
81
125
|
SET
|
82
126
|
#{updating_expression}
|
83
|
-
FROM source
|
127
|
+
FROM #{temp_source} AS source
|
84
128
|
WHERE #{matching_expression}
|
85
129
|
#{extra_conditions}
|
86
130
|
RETURNING target.*
|
87
131
|
)
|
88
132
|
INSERT INTO #{target_table} (#{target_fields})
|
89
133
|
SELECT #{target_fields}
|
90
|
-
FROM source
|
134
|
+
FROM #{temp_source} as source
|
91
135
|
WHERE NOT EXISTS (
|
92
136
|
SELECT 1
|
93
137
|
FROM updated target
|
94
138
|
WHERE #{matching_expression}
|
95
139
|
#{extra_conditions}
|
96
|
-
)
|
140
|
+
);
|
97
141
|
SQL
|
98
|
-
|
99
|
-
sql = model.send(:sanitize_sql, [sql, engine.common_params])
|
100
|
-
connection.execute sql
|
101
142
|
end
|
102
143
|
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
144
|
+
def insert_without_update_query(temp_source, target_fields)
|
145
|
+
<<-SQL
|
146
|
+
INSERT INTO #{target_table} (#{target_fields})
|
147
|
+
SELECT #{target_fields}
|
148
|
+
FROM #{temp_source} as source;
|
149
|
+
SQL
|
150
|
+
end
|
108
151
|
|
109
152
|
def selected_fields_expression
|
110
153
|
full_fields_map.map { |target_field, source_field| "#{source_field} as #{target_field}" }.join(', ')
|
@@ -133,6 +176,12 @@ module RedisCounters
|
|
133
176
|
result = conditions.map { |condition| "(#{condition})" }.join(' AND ')
|
134
177
|
result.present? ? "AND #{result}" : result
|
135
178
|
end
|
179
|
+
|
180
|
+
def source_conditions_expression
|
181
|
+
return if source_conditions.blank?
|
182
|
+
|
183
|
+
"WHERE #{source_conditions.map { |source_condition| "(#{source_condition})" }.join(' AND ')}"
|
184
|
+
end
|
136
185
|
end
|
137
186
|
end
|
138
187
|
end
|
@@ -59,7 +59,7 @@ module RedisCounters
|
|
59
59
|
# end
|
60
60
|
# end
|
61
61
|
#
|
62
|
-
# dumper.process!(counter, Date.yesterday)
|
62
|
+
# dumper.process!(counter, date: Date.yesterday)
|
63
63
|
#
|
64
64
|
# В результате все данные счетчика за вчера, будут
|
65
65
|
# смерджены в целевые таблицы, по ключевым полям: company_id и date,
|
@@ -100,7 +100,7 @@ module RedisCounters
|
|
100
100
|
attr_accessor :common_params
|
101
101
|
|
102
102
|
attr_reader :counter
|
103
|
-
attr_reader :
|
103
|
+
attr_reader :args
|
104
104
|
|
105
105
|
# callbacks
|
106
106
|
|
@@ -127,12 +127,13 @@ module RedisCounters
|
|
127
127
|
# Public: Производит перенос данных счетчика.
|
128
128
|
#
|
129
129
|
# counter - экземпляр счетчика.
|
130
|
-
#
|
130
|
+
# args - Hash - набор аргументов(кластер и/или партиции) для переноса данных.
|
131
131
|
#
|
132
132
|
# Returns Fixnum - кол-во обработанных строк.
|
133
133
|
#
|
134
|
-
def process!(counter,
|
135
|
-
@counter
|
134
|
+
def process!(counter, args = {})
|
135
|
+
@counter = counter
|
136
|
+
@args = args
|
136
137
|
|
137
138
|
db_transaction do
|
138
139
|
merge_data
|
@@ -185,7 +186,7 @@ module RedisCounters
|
|
185
186
|
end
|
186
187
|
|
187
188
|
def fill_temp_table
|
188
|
-
@rows_processed = counter.data(
|
189
|
+
@rows_processed = counter.data(args) do |batch|
|
189
190
|
@current_batch = batch
|
190
191
|
prepare_batch
|
191
192
|
insert_batch
|
@@ -225,8 +226,8 @@ module RedisCounters
|
|
225
226
|
|
226
227
|
def delete_from_redis
|
227
228
|
redis_session.pipelined do |redis|
|
228
|
-
counter.partitions(
|
229
|
-
counter.delete_partition_direct!(partition, redis)
|
229
|
+
counter.partitions(args).each do |partition|
|
230
|
+
counter.delete_partition_direct!(args.merge(partition), redis)
|
230
231
|
end
|
231
232
|
end
|
232
233
|
|
@@ -265,24 +266,20 @@ module RedisCounters
|
|
265
266
|
'character varying(4000)'
|
266
267
|
when :integer, :serial, :number
|
267
268
|
'integer'
|
268
|
-
when :date
|
269
|
-
|
270
|
-
when :timestamp
|
271
|
-
'timestamp'
|
272
|
-
when :boolean
|
273
|
-
'boolean'
|
269
|
+
when :date, :timestamp, :boolean, :hstore
|
270
|
+
type.to_s
|
274
271
|
else
|
275
|
-
|
272
|
+
if type.is_a?(Array) && type.first == :enum
|
273
|
+
type.last.fetch(:name)
|
274
|
+
else
|
275
|
+
raise 'Unknown datatype %s for %s field' % [type, field]
|
276
|
+
end
|
276
277
|
end
|
277
278
|
|
278
279
|
"#{field} #{pg_field_type}"
|
279
280
|
end.join(',')
|
280
281
|
end
|
281
282
|
|
282
|
-
def formatted_date
|
283
|
-
date.strftime(DATE_FORMAT)
|
284
|
-
end
|
285
|
-
|
286
283
|
def db_connection
|
287
284
|
destinations.first.connection
|
288
285
|
end
|
@@ -25,7 +25,6 @@ Gem::Specification.new do |spec|
|
|
25
25
|
spec.add_dependency 'callbacks_rb', '>= 0.0.1'
|
26
26
|
spec.add_dependency 'redis_counters', '>= 1.3'
|
27
27
|
|
28
|
-
|
29
28
|
spec.add_development_dependency 'bundler', '>= 1.7'
|
30
29
|
spec.add_development_dependency 'rake', '>= 10.0'
|
31
30
|
spec.add_development_dependency 'rspec', '>= 3.2'
|
data/spec/internal/db/schema.rb
CHANGED
@@ -1,9 +1,30 @@
|
|
1
1
|
ActiveRecord::Schema.define do
|
2
|
+
execute <<-SQL
|
3
|
+
CREATE TYPE subject_types AS ENUM ('');
|
4
|
+
SQL
|
5
|
+
|
6
|
+
if ::ActiveRecord::VERSION::MAJOR < 4
|
7
|
+
execute <<-SQL
|
8
|
+
CREATE EXTENSION IF NOT EXISTS hstore;
|
9
|
+
SQL
|
10
|
+
else
|
11
|
+
enable_extension :hstore
|
12
|
+
end
|
13
|
+
|
14
|
+
create_table :stats do |t|
|
15
|
+
t.integer :record_id, null: false
|
16
|
+
t.string :entity_type, null: false
|
17
|
+
t.datetime :date, null: false
|
18
|
+
t.hstore :params
|
19
|
+
end
|
20
|
+
|
2
21
|
create_table :stats_by_days do |t|
|
3
22
|
t.integer :record_id, null: false
|
4
23
|
t.integer :column_id, null: false
|
5
24
|
t.date :date, null: false
|
6
25
|
t.integer :hits, null: false, default: 0
|
26
|
+
t.column :subject, :subject_types
|
27
|
+
t.hstore :params
|
7
28
|
end
|
8
29
|
|
9
30
|
add_index :stats_by_days, [:record_id, :column_id, :date], unique: true
|
@@ -12,6 +33,8 @@ ActiveRecord::Schema.define do
|
|
12
33
|
t.integer :record_id, null: false
|
13
34
|
t.integer :column_id, null: false
|
14
35
|
t.integer :hits, null: false, default: 0
|
36
|
+
t.column :subject, :subject_types
|
37
|
+
t.hstore :params
|
15
38
|
end
|
16
39
|
|
17
40
|
add_index :stats_totals, [:record_id, :column_id], unique: true
|
@@ -19,6 +42,7 @@ ActiveRecord::Schema.define do
|
|
19
42
|
create_table :stats_agg_totals do |t|
|
20
43
|
t.integer :record_id, null: false
|
21
44
|
t.integer :hits, null: false, default: 0
|
45
|
+
t.column :subject, :subject_types
|
22
46
|
end
|
23
47
|
|
24
48
|
add_index :stats_agg_totals, [:record_id], unique: true
|
@@ -7,12 +7,14 @@ describe RedisCounters::Dumpers::Engine do
|
|
7
7
|
fields record_id: :integer,
|
8
8
|
column_id: :integer,
|
9
9
|
value: :integer,
|
10
|
-
date: :date
|
10
|
+
date: :date,
|
11
|
+
subject: [:enum, {name: :subject_types}],
|
12
|
+
params: :hstore
|
11
13
|
|
12
14
|
destination do
|
13
15
|
model StatsByDay
|
14
|
-
take :record_id, :column_id, :hits, :date
|
15
|
-
key_fields :record_id, :column_id, :date
|
16
|
+
take :record_id, :column_id, :hits, :date, :params
|
17
|
+
key_fields :record_id, :column_id, :date, :params
|
16
18
|
increment_fields :hits
|
17
19
|
map :hits, to: :value
|
18
20
|
condition 'target.date = :date'
|
@@ -20,8 +22,8 @@ describe RedisCounters::Dumpers::Engine do
|
|
20
22
|
|
21
23
|
destination do
|
22
24
|
model StatsTotal
|
23
|
-
take :record_id, :column_id, :hits
|
24
|
-
key_fields :record_id, :column_id
|
25
|
+
take :record_id, :column_id, :hits, :params
|
26
|
+
key_fields :record_id, :column_id, :params
|
25
27
|
increment_fields :hits
|
26
28
|
map :hits, to: :value
|
27
29
|
end
|
@@ -36,7 +38,7 @@ describe RedisCounters::Dumpers::Engine do
|
|
36
38
|
end
|
37
39
|
|
38
40
|
on_before_merge do |dumper, _connection|
|
39
|
-
dumper.common_params = {date: dumper.date.strftime('%Y-%m-%d')}
|
41
|
+
dumper.common_params = {date: dumper.args[:date].strftime('%Y-%m-%d')}
|
40
42
|
end
|
41
43
|
end
|
42
44
|
end
|
@@ -51,7 +53,7 @@ describe RedisCounters::Dumpers::Engine do
|
|
51
53
|
RedisCounters.create_counter(Redis.current,
|
52
54
|
counter_class: RedisCounters::HashCounter,
|
53
55
|
counter_name: :record_hits_by_day,
|
54
|
-
group_keys: [:record_id, :column_id],
|
56
|
+
group_keys: [:record_id, :column_id, :subject, :params],
|
55
57
|
partition_keys: [:date]
|
56
58
|
)
|
57
59
|
end
|
@@ -61,37 +63,156 @@ describe RedisCounters::Dumpers::Engine do
|
|
61
63
|
end
|
62
64
|
|
63
65
|
describe '#process!' do
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
66
|
+
context 'when increment_fields specified' do
|
67
|
+
before do
|
68
|
+
counter.increment(date: prev_date_s, record_id: 1, column_id: 100, subject: '', params: '')
|
69
|
+
counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
|
70
|
+
counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
|
71
|
+
counter.increment(date: prev_date_s, record_id: 2, column_id: 100, subject: nil, params: '')
|
69
72
|
|
70
|
-
|
73
|
+
params = {a: 1}.stringify_keys.to_s[1..-2]
|
74
|
+
counter.increment(date: prev_date_s, record_id: 3, column_id: 300, subject: nil, params: params)
|
71
75
|
|
72
|
-
|
73
|
-
counter.increment(date: date_s, record_id: 1, column_id: 200)
|
74
|
-
counter.increment(date: date_s, record_id: 1, column_id: 200)
|
75
|
-
counter.increment(date: date_s, record_id: 2, column_id: 100)
|
76
|
+
dumper.process!(counter, date: prev_date)
|
76
77
|
|
77
|
-
|
78
|
+
counter.increment(date: date_s, record_id: 1, column_id: 100, subject: '', params: '')
|
79
|
+
counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
|
80
|
+
counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
|
81
|
+
counter.increment(date: date_s, record_id: 2, column_id: 100, subject: nil, params: '')
|
82
|
+
|
83
|
+
dumper.process!(counter, date: date)
|
84
|
+
end
|
85
|
+
|
86
|
+
Then { expect(StatsByDay.count).to eq 7 }
|
87
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
|
88
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: prev_date).first.hits).to eq 2 }
|
89
|
+
And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
|
90
|
+
And { expect(StatsByDay.where(record_id: 3, column_id: 300, date: prev_date).first.params).to eq("a" => "1") }
|
91
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
|
92
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: date).first.hits).to eq 2 }
|
93
|
+
And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
|
94
|
+
|
95
|
+
And { expect(StatsTotal.count).to eq 4 }
|
96
|
+
And { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
|
97
|
+
And { expect(StatsTotal.where(record_id: 1, column_id: 200).first.hits).to eq 4 }
|
98
|
+
And { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
|
99
|
+
|
100
|
+
And { expect(StatsAggTotal.count).to eq 3 }
|
101
|
+
And { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 6 }
|
102
|
+
And { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
|
103
|
+
|
104
|
+
context 'with source conditions' do
|
105
|
+
let(:dumper) do
|
106
|
+
RedisCounters::Dumpers::Engine.build do
|
107
|
+
name :stats_totals
|
108
|
+
fields record_id: :integer,
|
109
|
+
column_id: :integer,
|
110
|
+
value: :integer,
|
111
|
+
date: :date
|
112
|
+
|
113
|
+
destination do
|
114
|
+
model StatsByDay
|
115
|
+
take :record_id, :column_id, :hits, :date
|
116
|
+
key_fields :record_id, :column_id, :date
|
117
|
+
increment_fields :hits
|
118
|
+
map :hits, to: :value
|
119
|
+
condition 'target.date = :date'
|
120
|
+
source_condition 'column_id = 100'
|
121
|
+
end
|
122
|
+
|
123
|
+
destination do
|
124
|
+
model StatsTotal
|
125
|
+
take :record_id, :column_id, :hits
|
126
|
+
key_fields :record_id, :column_id
|
127
|
+
increment_fields :hits
|
128
|
+
map :hits, to: :value
|
129
|
+
source_condition 'column_id = 100'
|
130
|
+
end
|
131
|
+
|
132
|
+
destination do
|
133
|
+
model StatsAggTotal
|
134
|
+
take :record_id, :hits
|
135
|
+
key_fields :record_id
|
136
|
+
increment_fields :hits
|
137
|
+
map :hits, to: 'sum(value)'
|
138
|
+
group_by :record_id
|
139
|
+
source_condition 'column_id = 100'
|
140
|
+
end
|
141
|
+
|
142
|
+
on_before_merge do |dumper, _connection|
|
143
|
+
dumper.common_params = {date: dumper.args[:date].strftime('%Y-%m-%d')}
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
Then { expect(StatsByDay.count).to eq 4 }
|
149
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
|
150
|
+
And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
|
151
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
|
152
|
+
And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
|
153
|
+
|
154
|
+
And { expect(StatsTotal.count).to eq 2 }
|
155
|
+
And { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
|
156
|
+
And { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
|
157
|
+
|
158
|
+
And { expect(StatsAggTotal.count).to eq 2 }
|
159
|
+
And { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 2 }
|
160
|
+
And { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
|
161
|
+
end
|
78
162
|
end
|
79
163
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
164
|
+
context 'when increment_fields not specified' do
|
165
|
+
let(:dumper) do
|
166
|
+
RedisCounters::Dumpers::Engine.build do
|
167
|
+
name :stats
|
168
|
+
fields record_id: :integer,
|
169
|
+
entity_type: :string,
|
170
|
+
date: :timestamp,
|
171
|
+
params: :hstore
|
172
|
+
|
173
|
+
destination do
|
174
|
+
model Stat
|
175
|
+
take :record_id, :entity_type, :date, :params
|
176
|
+
key_fields :record_id, :entity_type, :date, :params
|
177
|
+
end
|
178
|
+
|
179
|
+
on_before_merge do |dumper, _connection|
|
180
|
+
dumper.common_params = {entity_type: dumper.args[:entity_type]}
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
let(:counter) do
|
186
|
+
RedisCounters.create_counter(
|
187
|
+
Redis.current,
|
188
|
+
counter_class: RedisCounters::HashCounter,
|
189
|
+
counter_name: :all_stats,
|
190
|
+
value_delimiter: ';',
|
191
|
+
group_keys: [:date, :record_id, :params],
|
192
|
+
partition_keys: [:entity_type]
|
193
|
+
)
|
194
|
+
end
|
195
|
+
|
196
|
+
let(:date) { Time.now.utc }
|
197
|
+
|
198
|
+
before do
|
199
|
+
counter.increment(entity_type: 'Type1', date: date, record_id: 1, params: '')
|
200
|
+
counter.increment(entity_type: 'Type2', date: date, record_id: 1, params: '')
|
201
|
+
counter.increment(entity_type: 'Type1', date: date - 1.minute, record_id: 1, params: '')
|
202
|
+
counter.increment(entity_type: 'Type1', date: date - 10.minutes, record_id: 1, params: '')
|
203
|
+
counter.increment(entity_type: 'Type1', date: date, record_id: 2, params: '')
|
204
|
+
|
205
|
+
params = {a: 1}.stringify_keys.to_s[1..-2]
|
206
|
+
counter.increment(entity_type: 'Type1', date: date, record_id: 3, params: params)
|
207
|
+
|
208
|
+
dumper.process!(counter, entity_type: 'Type1')
|
209
|
+
dumper.process!(counter, entity_type: 'Type2')
|
210
|
+
end
|
211
|
+
|
212
|
+
Then { expect(Stat.count).to eq 6 }
|
213
|
+
And { expect(Stat.where(entity_type: 'Type1').count).to eq 5 }
|
214
|
+
And { expect(Stat.where(entity_type: 'Type2').count).to eq 1 }
|
215
|
+
And { expect(Stat.where(record_id: 3, entity_type: 'Type1').first.params).to eq("a" => "1") }
|
216
|
+
end
|
96
217
|
end
|
97
218
|
end
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,17 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redis_counters-dumpers
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Merkushin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-10-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
prerelease: false
|
15
14
|
version_requirements: !ruby/object:Gem::Requirement
|
16
15
|
requirements:
|
17
16
|
- - '>='
|
@@ -24,8 +23,8 @@ dependencies:
|
|
24
23
|
version: '3.0'
|
25
24
|
type: :runtime
|
26
25
|
name: activesupport
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
26
|
prerelease: false
|
27
|
+
- !ruby/object:Gem::Dependency
|
29
28
|
version_requirements: !ruby/object:Gem::Requirement
|
30
29
|
requirements:
|
31
30
|
- - '>='
|
@@ -38,8 +37,8 @@ dependencies:
|
|
38
37
|
version: '3.0'
|
39
38
|
type: :runtime
|
40
39
|
name: activerecord
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
40
|
prerelease: false
|
41
|
+
- !ruby/object:Gem::Dependency
|
43
42
|
version_requirements: !ruby/object:Gem::Requirement
|
44
43
|
requirements:
|
45
44
|
- - '>='
|
@@ -52,8 +51,8 @@ dependencies:
|
|
52
51
|
version: '0'
|
53
52
|
type: :runtime
|
54
53
|
name: pg
|
55
|
-
- !ruby/object:Gem::Dependency
|
56
54
|
prerelease: false
|
55
|
+
- !ruby/object:Gem::Dependency
|
57
56
|
version_requirements: !ruby/object:Gem::Requirement
|
58
57
|
requirements:
|
59
58
|
- - '>='
|
@@ -66,8 +65,8 @@ dependencies:
|
|
66
65
|
version: '3.0'
|
67
66
|
type: :runtime
|
68
67
|
name: redis
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
68
|
prerelease: false
|
69
|
+
- !ruby/object:Gem::Dependency
|
71
70
|
version_requirements: !ruby/object:Gem::Requirement
|
72
71
|
requirements:
|
73
72
|
- - '>='
|
@@ -80,8 +79,8 @@ dependencies:
|
|
80
79
|
version: '1.3'
|
81
80
|
type: :runtime
|
82
81
|
name: redis-namespace
|
83
|
-
- !ruby/object:Gem::Dependency
|
84
82
|
prerelease: false
|
83
|
+
- !ruby/object:Gem::Dependency
|
85
84
|
version_requirements: !ruby/object:Gem::Requirement
|
86
85
|
requirements:
|
87
86
|
- - '>='
|
@@ -94,8 +93,8 @@ dependencies:
|
|
94
93
|
version: 0.0.1
|
95
94
|
type: :runtime
|
96
95
|
name: callbacks_rb
|
97
|
-
- !ruby/object:Gem::Dependency
|
98
96
|
prerelease: false
|
97
|
+
- !ruby/object:Gem::Dependency
|
99
98
|
version_requirements: !ruby/object:Gem::Requirement
|
100
99
|
requirements:
|
101
100
|
- - '>='
|
@@ -108,8 +107,8 @@ dependencies:
|
|
108
107
|
version: '1.3'
|
109
108
|
type: :runtime
|
110
109
|
name: redis_counters
|
111
|
-
- !ruby/object:Gem::Dependency
|
112
110
|
prerelease: false
|
111
|
+
- !ruby/object:Gem::Dependency
|
113
112
|
version_requirements: !ruby/object:Gem::Requirement
|
114
113
|
requirements:
|
115
114
|
- - '>='
|
@@ -122,8 +121,8 @@ dependencies:
|
|
122
121
|
version: '1.7'
|
123
122
|
type: :development
|
124
123
|
name: bundler
|
125
|
-
- !ruby/object:Gem::Dependency
|
126
124
|
prerelease: false
|
125
|
+
- !ruby/object:Gem::Dependency
|
127
126
|
version_requirements: !ruby/object:Gem::Requirement
|
128
127
|
requirements:
|
129
128
|
- - '>='
|
@@ -136,8 +135,8 @@ dependencies:
|
|
136
135
|
version: '10.0'
|
137
136
|
type: :development
|
138
137
|
name: rake
|
139
|
-
- !ruby/object:Gem::Dependency
|
140
138
|
prerelease: false
|
139
|
+
- !ruby/object:Gem::Dependency
|
141
140
|
version_requirements: !ruby/object:Gem::Requirement
|
142
141
|
requirements:
|
143
142
|
- - '>='
|
@@ -150,8 +149,8 @@ dependencies:
|
|
150
149
|
version: '3.2'
|
151
150
|
type: :development
|
152
151
|
name: rspec
|
153
|
-
- !ruby/object:Gem::Dependency
|
154
152
|
prerelease: false
|
153
|
+
- !ruby/object:Gem::Dependency
|
155
154
|
version_requirements: !ruby/object:Gem::Requirement
|
156
155
|
requirements:
|
157
156
|
- - '>='
|
@@ -164,8 +163,8 @@ dependencies:
|
|
164
163
|
version: '3.2'
|
165
164
|
type: :development
|
166
165
|
name: rspec-rails
|
167
|
-
- !ruby/object:Gem::Dependency
|
168
166
|
prerelease: false
|
167
|
+
- !ruby/object:Gem::Dependency
|
169
168
|
version_requirements: !ruby/object:Gem::Requirement
|
170
169
|
requirements:
|
171
170
|
- - '>='
|
@@ -178,8 +177,8 @@ dependencies:
|
|
178
177
|
version: '3.5'
|
179
178
|
type: :development
|
180
179
|
name: rspec-given
|
181
|
-
- !ruby/object:Gem::Dependency
|
182
180
|
prerelease: false
|
181
|
+
- !ruby/object:Gem::Dependency
|
183
182
|
version_requirements: !ruby/object:Gem::Requirement
|
184
183
|
requirements:
|
185
184
|
- - '>='
|
@@ -192,8 +191,8 @@ dependencies:
|
|
192
191
|
version: 1.0.2
|
193
192
|
type: :development
|
194
193
|
name: appraisal
|
195
|
-
- !ruby/object:Gem::Dependency
|
196
194
|
prerelease: false
|
195
|
+
- !ruby/object:Gem::Dependency
|
197
196
|
version_requirements: !ruby/object:Gem::Requirement
|
198
197
|
requirements:
|
199
198
|
- - '>='
|
@@ -206,8 +205,8 @@ dependencies:
|
|
206
205
|
version: '0'
|
207
206
|
type: :development
|
208
207
|
name: mock_redis
|
209
|
-
- !ruby/object:Gem::Dependency
|
210
208
|
prerelease: false
|
209
|
+
- !ruby/object:Gem::Dependency
|
211
210
|
version_requirements: !ruby/object:Gem::Requirement
|
212
211
|
requirements:
|
213
212
|
- - '>='
|
@@ -220,6 +219,7 @@ dependencies:
|
|
220
219
|
version: '0'
|
221
220
|
type: :development
|
222
221
|
name: apress-changelogger
|
222
|
+
prerelease: false
|
223
223
|
description:
|
224
224
|
email:
|
225
225
|
- bibendi@bk.ru
|
@@ -230,6 +230,7 @@ files:
|
|
230
230
|
- .gitignore
|
231
231
|
- .rspec
|
232
232
|
- Appraisals
|
233
|
+
- CHANGELOG.md
|
233
234
|
- Gemfile
|
234
235
|
- LICENSE.txt
|
235
236
|
- Makefile
|
@@ -250,6 +251,7 @@ files:
|
|
250
251
|
- lib/redis_counters/dumpers/list.rb
|
251
252
|
- lib/redis_counters/dumpers/version.rb
|
252
253
|
- redis_counters-dumpers.gemspec
|
254
|
+
- spec/internal/app/models/stat.rb
|
253
255
|
- spec/internal/app/models/stats_agg_total.rb
|
254
256
|
- spec/internal/app/models/stats_by_day.rb
|
255
257
|
- spec/internal/app/models/stats_total.rb
|
@@ -278,11 +280,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
278
280
|
version: '0'
|
279
281
|
requirements: []
|
280
282
|
rubyforge_project:
|
281
|
-
rubygems_version: 2.
|
283
|
+
rubygems_version: 2.6.1
|
282
284
|
signing_key:
|
283
285
|
specification_version: 4
|
284
286
|
summary: Dump statistics from Redis to DB
|
285
287
|
test_files:
|
288
|
+
- spec/internal/app/models/stat.rb
|
286
289
|
- spec/internal/app/models/stats_agg_total.rb
|
287
290
|
- spec/internal/app/models/stats_by_day.rb
|
288
291
|
- spec/internal/app/models/stats_total.rb
|