redis_counters-dumpers 0.1.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/Gemfile +4 -0
- data/lib/redis_counters/dumpers/destination.rb +67 -18
- data/lib/redis_counters/dumpers/dsl/destination.rb +4 -0
- data/lib/redis_counters/dumpers/engine.rb +16 -19
- data/lib/redis_counters/dumpers/version.rb +1 -1
- data/redis_counters-dumpers.gemspec +0 -1
- data/spec/internal/app/models/stat.rb +3 -0
- data/spec/internal/app/models/stats_by_day.rb +1 -0
- data/spec/internal/app/models/stats_total.rb +1 -0
- data/spec/internal/db/schema.rb +24 -0
- data/spec/lib/redis_counters/dumpers/engine_spec.rb +155 -34
- data/spec/spec_helper.rb +5 -0
- metadata +21 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: baf28efb0b4f140dae575fd7aa300a1dbb1486ec
|
4
|
+
data.tar.gz: e525a5a8049f12af5c7985a77dc6748901b64181
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d808c8e16e3438efc682c06161a51d05612fb02c3fbb286f1cd68fa714eedc25d4be52b76b0b4cae3a579b36c07726abd9946f02f11195ae18958e6535a2c4ae
|
7
|
+
data.tar.gz: d51ac56268477487b776b17fa583b5a5ab020af09acd54ae445d42f3ad4f8c94d34917da02585b2b870f8c0a3c49343579b1de69fc393ab21513d96079acde05
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
|
2
|
+
#### [Current]
|
3
|
+
* 2015-09-10 [1f96c39](../../commit/1f96c39) - __(Zhidkov Denis)__ feat: add source_conditions method
|
4
|
+
* 2015-02-19 [8885300](../../commit/8885300) - __(bibendi)__ add group_by method
|
5
|
+
|
6
|
+
#### v0.0.1
|
7
|
+
* 2015-01-26 [86918bd](../../commit/86918bd) - __(bibendi)__ initial code base
|
8
|
+
* 2015-01-26 [a238032](../../commit/a238032) - __(bibendi)__ initial
|
data/Gemfile
CHANGED
@@ -4,6 +4,10 @@ source 'https://rubygems.org'
|
|
4
4
|
|
5
5
|
group :development, :test do
|
6
6
|
gem 'combustion', github: 'pat/combustion', ref: '7d0d24c3f36ce0eb336177fc493be0721bc26665'
|
7
|
+
gem 'activerecord-postgres-hstore', require: false
|
8
|
+
gem 'simple_hstore_accessor', '~> 0.2', require: false
|
7
9
|
end
|
8
10
|
|
11
|
+
gem 'rack', '< 2' if RUBY_VERSION < '2.2.0'
|
12
|
+
|
9
13
|
gemspec
|
@@ -58,53 +58,96 @@ module RedisCounters
|
|
58
58
|
# Условия соеденяются через AND.
|
59
59
|
attr_accessor :conditions
|
60
60
|
|
61
|
+
# Список дополнительных условий, которые применяются для выборки из source-таблицы для обновления
|
62
|
+
# target, Array of String.
|
63
|
+
# Каждое условие представляет собой строку - часть SQL выражения, которое может включать именованные
|
64
|
+
# параметры из числа доступных в хеше общих параметров дампера: engine.common_params.
|
65
|
+
# Условия соединяются через AND.
|
66
|
+
attr_accessor :source_conditions
|
67
|
+
|
61
68
|
def initialize(engine)
|
62
69
|
@engine = engine
|
63
70
|
@fields_map = HashWithIndifferentAccess.new
|
64
71
|
@conditions = []
|
72
|
+
@source_conditions = []
|
65
73
|
end
|
66
74
|
|
67
75
|
def merge
|
76
|
+
sql = generate_query
|
77
|
+
sql = model.send(:sanitize_sql, [sql, engine.common_params])
|
78
|
+
connection.execute sql
|
79
|
+
end
|
80
|
+
|
81
|
+
def_delegator :model, :connection
|
82
|
+
def_delegator :model, :quoted_table_name, :target_table
|
83
|
+
def_delegator :engine, :temp_table_name, :source_table
|
84
|
+
|
85
|
+
protected
|
86
|
+
|
87
|
+
def generate_query
|
68
88
|
target_fields = fields.join(', ')
|
89
|
+
temp_source = "_source_#{source_table}"
|
90
|
+
|
91
|
+
query = create_temp_table_query(temp_source)
|
92
|
+
|
93
|
+
if increment_fields.present?
|
94
|
+
query.concat(insert_with_update_query(temp_source, target_fields))
|
95
|
+
else
|
96
|
+
query.concat(insert_without_update_query(temp_source, target_fields))
|
97
|
+
end
|
98
|
+
|
99
|
+
query.concat(drop_temp_table_query(temp_source))
|
100
|
+
query
|
101
|
+
end
|
102
|
+
|
103
|
+
def create_temp_table_query(temp_source)
|
104
|
+
<<-SQL
|
105
|
+
CREATE TEMP TABLE #{temp_source} ON COMMIT DROP AS
|
106
|
+
SELECT #{selected_fields_expression}
|
107
|
+
FROM #{source_table}
|
108
|
+
#{source_conditions_expression}
|
109
|
+
#{group_by_expression};
|
110
|
+
SQL
|
111
|
+
end
|
69
112
|
|
70
|
-
|
113
|
+
def drop_temp_table_query(temp_source)
|
114
|
+
<<-SQL
|
115
|
+
DROP TABLE #{temp_source};
|
116
|
+
SQL
|
117
|
+
end
|
118
|
+
|
119
|
+
def insert_with_update_query(temp_source, target_fields)
|
120
|
+
<<-SQL
|
71
121
|
WITH
|
72
|
-
source AS
|
73
|
-
(
|
74
|
-
SELECT #{selected_fields_expression}
|
75
|
-
FROM #{source_table}
|
76
|
-
#{group_by_expression}
|
77
|
-
),
|
78
122
|
updated AS
|
79
123
|
(
|
80
124
|
UPDATE #{target_table} target
|
81
125
|
SET
|
82
126
|
#{updating_expression}
|
83
|
-
FROM source
|
127
|
+
FROM #{temp_source} AS source
|
84
128
|
WHERE #{matching_expression}
|
85
129
|
#{extra_conditions}
|
86
130
|
RETURNING target.*
|
87
131
|
)
|
88
132
|
INSERT INTO #{target_table} (#{target_fields})
|
89
133
|
SELECT #{target_fields}
|
90
|
-
FROM source
|
134
|
+
FROM #{temp_source} as source
|
91
135
|
WHERE NOT EXISTS (
|
92
136
|
SELECT 1
|
93
137
|
FROM updated target
|
94
138
|
WHERE #{matching_expression}
|
95
139
|
#{extra_conditions}
|
96
|
-
)
|
140
|
+
);
|
97
141
|
SQL
|
98
|
-
|
99
|
-
sql = model.send(:sanitize_sql, [sql, engine.common_params])
|
100
|
-
connection.execute sql
|
101
142
|
end
|
102
143
|
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
144
|
+
def insert_without_update_query(temp_source, target_fields)
|
145
|
+
<<-SQL
|
146
|
+
INSERT INTO #{target_table} (#{target_fields})
|
147
|
+
SELECT #{target_fields}
|
148
|
+
FROM #{temp_source} as source;
|
149
|
+
SQL
|
150
|
+
end
|
108
151
|
|
109
152
|
def selected_fields_expression
|
110
153
|
full_fields_map.map { |target_field, source_field| "#{source_field} as #{target_field}" }.join(', ')
|
@@ -133,6 +176,12 @@ module RedisCounters
|
|
133
176
|
result = conditions.map { |condition| "(#{condition})" }.join(' AND ')
|
134
177
|
result.present? ? "AND #{result}" : result
|
135
178
|
end
|
179
|
+
|
180
|
+
def source_conditions_expression
|
181
|
+
return if source_conditions.blank?
|
182
|
+
|
183
|
+
"WHERE #{source_conditions.map { |source_condition| "(#{source_condition})" }.join(' AND ')}"
|
184
|
+
end
|
136
185
|
end
|
137
186
|
end
|
138
187
|
end
|
@@ -59,7 +59,7 @@ module RedisCounters
|
|
59
59
|
# end
|
60
60
|
# end
|
61
61
|
#
|
62
|
-
# dumper.process!(counter, Date.yesterday)
|
62
|
+
# dumper.process!(counter, date: Date.yesterday)
|
63
63
|
#
|
64
64
|
# В результате все данные счетчика за вчера, будут
|
65
65
|
# смерджены в целевые таблицы, по ключевым полям: company_id и date,
|
@@ -100,7 +100,7 @@ module RedisCounters
|
|
100
100
|
attr_accessor :common_params
|
101
101
|
|
102
102
|
attr_reader :counter
|
103
|
-
attr_reader :
|
103
|
+
attr_reader :args
|
104
104
|
|
105
105
|
# callbacks
|
106
106
|
|
@@ -127,12 +127,13 @@ module RedisCounters
|
|
127
127
|
# Public: Производит перенос данных счетчика.
|
128
128
|
#
|
129
129
|
# counter - экземпляр счетчика.
|
130
|
-
#
|
130
|
+
# args - Hash - набор аргументов(кластер и/или партиции) для переноса данных.
|
131
131
|
#
|
132
132
|
# Returns Fixnum - кол-во обработанных строк.
|
133
133
|
#
|
134
|
-
def process!(counter,
|
135
|
-
@counter
|
134
|
+
def process!(counter, args = {})
|
135
|
+
@counter = counter
|
136
|
+
@args = args
|
136
137
|
|
137
138
|
db_transaction do
|
138
139
|
merge_data
|
@@ -185,7 +186,7 @@ module RedisCounters
|
|
185
186
|
end
|
186
187
|
|
187
188
|
def fill_temp_table
|
188
|
-
@rows_processed = counter.data(
|
189
|
+
@rows_processed = counter.data(args) do |batch|
|
189
190
|
@current_batch = batch
|
190
191
|
prepare_batch
|
191
192
|
insert_batch
|
@@ -225,8 +226,8 @@ module RedisCounters
|
|
225
226
|
|
226
227
|
def delete_from_redis
|
227
228
|
redis_session.pipelined do |redis|
|
228
|
-
counter.partitions(
|
229
|
-
counter.delete_partition_direct!(partition, redis)
|
229
|
+
counter.partitions(args).each do |partition|
|
230
|
+
counter.delete_partition_direct!(args.merge(partition), redis)
|
230
231
|
end
|
231
232
|
end
|
232
233
|
|
@@ -265,24 +266,20 @@ module RedisCounters
|
|
265
266
|
'character varying(4000)'
|
266
267
|
when :integer, :serial, :number
|
267
268
|
'integer'
|
268
|
-
when :date
|
269
|
-
|
270
|
-
when :timestamp
|
271
|
-
'timestamp'
|
272
|
-
when :boolean
|
273
|
-
'boolean'
|
269
|
+
when :date, :timestamp, :boolean, :hstore
|
270
|
+
type.to_s
|
274
271
|
else
|
275
|
-
|
272
|
+
if type.is_a?(Array) && type.first == :enum
|
273
|
+
type.last.fetch(:name)
|
274
|
+
else
|
275
|
+
raise 'Unknown datatype %s for %s field' % [type, field]
|
276
|
+
end
|
276
277
|
end
|
277
278
|
|
278
279
|
"#{field} #{pg_field_type}"
|
279
280
|
end.join(',')
|
280
281
|
end
|
281
282
|
|
282
|
-
def formatted_date
|
283
|
-
date.strftime(DATE_FORMAT)
|
284
|
-
end
|
285
|
-
|
286
283
|
def db_connection
|
287
284
|
destinations.first.connection
|
288
285
|
end
|
@@ -25,7 +25,6 @@ Gem::Specification.new do |spec|
|
|
25
25
|
spec.add_dependency 'callbacks_rb', '>= 0.0.1'
|
26
26
|
spec.add_dependency 'redis_counters', '>= 1.3'
|
27
27
|
|
28
|
-
|
29
28
|
spec.add_development_dependency 'bundler', '>= 1.7'
|
30
29
|
spec.add_development_dependency 'rake', '>= 10.0'
|
31
30
|
spec.add_development_dependency 'rspec', '>= 3.2'
|
data/spec/internal/db/schema.rb
CHANGED
@@ -1,9 +1,30 @@
|
|
1
1
|
ActiveRecord::Schema.define do
|
2
|
+
execute <<-SQL
|
3
|
+
CREATE TYPE subject_types AS ENUM ('');
|
4
|
+
SQL
|
5
|
+
|
6
|
+
if ::ActiveRecord::VERSION::MAJOR < 4
|
7
|
+
execute <<-SQL
|
8
|
+
CREATE EXTENSION IF NOT EXISTS hstore;
|
9
|
+
SQL
|
10
|
+
else
|
11
|
+
enable_extension :hstore
|
12
|
+
end
|
13
|
+
|
14
|
+
create_table :stats do |t|
|
15
|
+
t.integer :record_id, null: false
|
16
|
+
t.string :entity_type, null: false
|
17
|
+
t.datetime :date, null: false
|
18
|
+
t.hstore :params
|
19
|
+
end
|
20
|
+
|
2
21
|
create_table :stats_by_days do |t|
|
3
22
|
t.integer :record_id, null: false
|
4
23
|
t.integer :column_id, null: false
|
5
24
|
t.date :date, null: false
|
6
25
|
t.integer :hits, null: false, default: 0
|
26
|
+
t.column :subject, :subject_types
|
27
|
+
t.hstore :params
|
7
28
|
end
|
8
29
|
|
9
30
|
add_index :stats_by_days, [:record_id, :column_id, :date], unique: true
|
@@ -12,6 +33,8 @@ ActiveRecord::Schema.define do
|
|
12
33
|
t.integer :record_id, null: false
|
13
34
|
t.integer :column_id, null: false
|
14
35
|
t.integer :hits, null: false, default: 0
|
36
|
+
t.column :subject, :subject_types
|
37
|
+
t.hstore :params
|
15
38
|
end
|
16
39
|
|
17
40
|
add_index :stats_totals, [:record_id, :column_id], unique: true
|
@@ -19,6 +42,7 @@ ActiveRecord::Schema.define do
|
|
19
42
|
create_table :stats_agg_totals do |t|
|
20
43
|
t.integer :record_id, null: false
|
21
44
|
t.integer :hits, null: false, default: 0
|
45
|
+
t.column :subject, :subject_types
|
22
46
|
end
|
23
47
|
|
24
48
|
add_index :stats_agg_totals, [:record_id], unique: true
|
@@ -7,12 +7,14 @@ describe RedisCounters::Dumpers::Engine do
|
|
7
7
|
fields record_id: :integer,
|
8
8
|
column_id: :integer,
|
9
9
|
value: :integer,
|
10
|
-
date: :date
|
10
|
+
date: :date,
|
11
|
+
subject: [:enum, {name: :subject_types}],
|
12
|
+
params: :hstore
|
11
13
|
|
12
14
|
destination do
|
13
15
|
model StatsByDay
|
14
|
-
take :record_id, :column_id, :hits, :date
|
15
|
-
key_fields :record_id, :column_id, :date
|
16
|
+
take :record_id, :column_id, :hits, :date, :params
|
17
|
+
key_fields :record_id, :column_id, :date, :params
|
16
18
|
increment_fields :hits
|
17
19
|
map :hits, to: :value
|
18
20
|
condition 'target.date = :date'
|
@@ -20,8 +22,8 @@ describe RedisCounters::Dumpers::Engine do
|
|
20
22
|
|
21
23
|
destination do
|
22
24
|
model StatsTotal
|
23
|
-
take :record_id, :column_id, :hits
|
24
|
-
key_fields :record_id, :column_id
|
25
|
+
take :record_id, :column_id, :hits, :params
|
26
|
+
key_fields :record_id, :column_id, :params
|
25
27
|
increment_fields :hits
|
26
28
|
map :hits, to: :value
|
27
29
|
end
|
@@ -36,7 +38,7 @@ describe RedisCounters::Dumpers::Engine do
|
|
36
38
|
end
|
37
39
|
|
38
40
|
on_before_merge do |dumper, _connection|
|
39
|
-
dumper.common_params = {date: dumper.date.strftime('%Y-%m-%d')}
|
41
|
+
dumper.common_params = {date: dumper.args[:date].strftime('%Y-%m-%d')}
|
40
42
|
end
|
41
43
|
end
|
42
44
|
end
|
@@ -51,7 +53,7 @@ describe RedisCounters::Dumpers::Engine do
|
|
51
53
|
RedisCounters.create_counter(Redis.current,
|
52
54
|
counter_class: RedisCounters::HashCounter,
|
53
55
|
counter_name: :record_hits_by_day,
|
54
|
-
group_keys: [:record_id, :column_id],
|
56
|
+
group_keys: [:record_id, :column_id, :subject, :params],
|
55
57
|
partition_keys: [:date]
|
56
58
|
)
|
57
59
|
end
|
@@ -61,37 +63,156 @@ describe RedisCounters::Dumpers::Engine do
|
|
61
63
|
end
|
62
64
|
|
63
65
|
describe '#process!' do
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
66
|
+
context 'when increment_fields specified' do
|
67
|
+
before do
|
68
|
+
counter.increment(date: prev_date_s, record_id: 1, column_id: 100, subject: '', params: '')
|
69
|
+
counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
|
70
|
+
counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
|
71
|
+
counter.increment(date: prev_date_s, record_id: 2, column_id: 100, subject: nil, params: '')
|
69
72
|
|
70
|
-
|
73
|
+
params = {a: 1}.stringify_keys.to_s[1..-2]
|
74
|
+
counter.increment(date: prev_date_s, record_id: 3, column_id: 300, subject: nil, params: params)
|
71
75
|
|
72
|
-
|
73
|
-
counter.increment(date: date_s, record_id: 1, column_id: 200)
|
74
|
-
counter.increment(date: date_s, record_id: 1, column_id: 200)
|
75
|
-
counter.increment(date: date_s, record_id: 2, column_id: 100)
|
76
|
+
dumper.process!(counter, date: prev_date)
|
76
77
|
|
77
|
-
|
78
|
+
counter.increment(date: date_s, record_id: 1, column_id: 100, subject: '', params: '')
|
79
|
+
counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
|
80
|
+
counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
|
81
|
+
counter.increment(date: date_s, record_id: 2, column_id: 100, subject: nil, params: '')
|
82
|
+
|
83
|
+
dumper.process!(counter, date: date)
|
84
|
+
end
|
85
|
+
|
86
|
+
Then { expect(StatsByDay.count).to eq 7 }
|
87
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
|
88
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: prev_date).first.hits).to eq 2 }
|
89
|
+
And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
|
90
|
+
And { expect(StatsByDay.where(record_id: 3, column_id: 300, date: prev_date).first.params).to eq("a" => "1") }
|
91
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
|
92
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: date).first.hits).to eq 2 }
|
93
|
+
And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
|
94
|
+
|
95
|
+
And { expect(StatsTotal.count).to eq 4 }
|
96
|
+
And { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
|
97
|
+
And { expect(StatsTotal.where(record_id: 1, column_id: 200).first.hits).to eq 4 }
|
98
|
+
And { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
|
99
|
+
|
100
|
+
And { expect(StatsAggTotal.count).to eq 3 }
|
101
|
+
And { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 6 }
|
102
|
+
And { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
|
103
|
+
|
104
|
+
context 'with source conditions' do
|
105
|
+
let(:dumper) do
|
106
|
+
RedisCounters::Dumpers::Engine.build do
|
107
|
+
name :stats_totals
|
108
|
+
fields record_id: :integer,
|
109
|
+
column_id: :integer,
|
110
|
+
value: :integer,
|
111
|
+
date: :date
|
112
|
+
|
113
|
+
destination do
|
114
|
+
model StatsByDay
|
115
|
+
take :record_id, :column_id, :hits, :date
|
116
|
+
key_fields :record_id, :column_id, :date
|
117
|
+
increment_fields :hits
|
118
|
+
map :hits, to: :value
|
119
|
+
condition 'target.date = :date'
|
120
|
+
source_condition 'column_id = 100'
|
121
|
+
end
|
122
|
+
|
123
|
+
destination do
|
124
|
+
model StatsTotal
|
125
|
+
take :record_id, :column_id, :hits
|
126
|
+
key_fields :record_id, :column_id
|
127
|
+
increment_fields :hits
|
128
|
+
map :hits, to: :value
|
129
|
+
source_condition 'column_id = 100'
|
130
|
+
end
|
131
|
+
|
132
|
+
destination do
|
133
|
+
model StatsAggTotal
|
134
|
+
take :record_id, :hits
|
135
|
+
key_fields :record_id
|
136
|
+
increment_fields :hits
|
137
|
+
map :hits, to: 'sum(value)'
|
138
|
+
group_by :record_id
|
139
|
+
source_condition 'column_id = 100'
|
140
|
+
end
|
141
|
+
|
142
|
+
on_before_merge do |dumper, _connection|
|
143
|
+
dumper.common_params = {date: dumper.args[:date].strftime('%Y-%m-%d')}
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
Then { expect(StatsByDay.count).to eq 4 }
|
149
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
|
150
|
+
And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
|
151
|
+
And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
|
152
|
+
And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
|
153
|
+
|
154
|
+
And { expect(StatsTotal.count).to eq 2 }
|
155
|
+
And { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
|
156
|
+
And { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
|
157
|
+
|
158
|
+
And { expect(StatsAggTotal.count).to eq 2 }
|
159
|
+
And { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 2 }
|
160
|
+
And { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
|
161
|
+
end
|
78
162
|
end
|
79
163
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
164
|
+
context 'when increment_fields not specified' do
|
165
|
+
let(:dumper) do
|
166
|
+
RedisCounters::Dumpers::Engine.build do
|
167
|
+
name :stats
|
168
|
+
fields record_id: :integer,
|
169
|
+
entity_type: :string,
|
170
|
+
date: :timestamp,
|
171
|
+
params: :hstore
|
172
|
+
|
173
|
+
destination do
|
174
|
+
model Stat
|
175
|
+
take :record_id, :entity_type, :date, :params
|
176
|
+
key_fields :record_id, :entity_type, :date, :params
|
177
|
+
end
|
178
|
+
|
179
|
+
on_before_merge do |dumper, _connection|
|
180
|
+
dumper.common_params = {entity_type: dumper.args[:entity_type]}
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
let(:counter) do
|
186
|
+
RedisCounters.create_counter(
|
187
|
+
Redis.current,
|
188
|
+
counter_class: RedisCounters::HashCounter,
|
189
|
+
counter_name: :all_stats,
|
190
|
+
value_delimiter: ';',
|
191
|
+
group_keys: [:date, :record_id, :params],
|
192
|
+
partition_keys: [:entity_type]
|
193
|
+
)
|
194
|
+
end
|
195
|
+
|
196
|
+
let(:date) { Time.now.utc }
|
197
|
+
|
198
|
+
before do
|
199
|
+
counter.increment(entity_type: 'Type1', date: date, record_id: 1, params: '')
|
200
|
+
counter.increment(entity_type: 'Type2', date: date, record_id: 1, params: '')
|
201
|
+
counter.increment(entity_type: 'Type1', date: date - 1.minute, record_id: 1, params: '')
|
202
|
+
counter.increment(entity_type: 'Type1', date: date - 10.minutes, record_id: 1, params: '')
|
203
|
+
counter.increment(entity_type: 'Type1', date: date, record_id: 2, params: '')
|
204
|
+
|
205
|
+
params = {a: 1}.stringify_keys.to_s[1..-2]
|
206
|
+
counter.increment(entity_type: 'Type1', date: date, record_id: 3, params: params)
|
207
|
+
|
208
|
+
dumper.process!(counter, entity_type: 'Type1')
|
209
|
+
dumper.process!(counter, entity_type: 'Type2')
|
210
|
+
end
|
211
|
+
|
212
|
+
Then { expect(Stat.count).to eq 6 }
|
213
|
+
And { expect(Stat.where(entity_type: 'Type1').count).to eq 5 }
|
214
|
+
And { expect(Stat.where(entity_type: 'Type2').count).to eq 1 }
|
215
|
+
And { expect(Stat.where(record_id: 3, entity_type: 'Type1').first.params).to eq("a" => "1") }
|
216
|
+
end
|
96
217
|
end
|
97
218
|
end
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,17 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redis_counters-dumpers
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Merkushin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-10-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
prerelease: false
|
15
14
|
version_requirements: !ruby/object:Gem::Requirement
|
16
15
|
requirements:
|
17
16
|
- - '>='
|
@@ -24,8 +23,8 @@ dependencies:
|
|
24
23
|
version: '3.0'
|
25
24
|
type: :runtime
|
26
25
|
name: activesupport
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
26
|
prerelease: false
|
27
|
+
- !ruby/object:Gem::Dependency
|
29
28
|
version_requirements: !ruby/object:Gem::Requirement
|
30
29
|
requirements:
|
31
30
|
- - '>='
|
@@ -38,8 +37,8 @@ dependencies:
|
|
38
37
|
version: '3.0'
|
39
38
|
type: :runtime
|
40
39
|
name: activerecord
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
40
|
prerelease: false
|
41
|
+
- !ruby/object:Gem::Dependency
|
43
42
|
version_requirements: !ruby/object:Gem::Requirement
|
44
43
|
requirements:
|
45
44
|
- - '>='
|
@@ -52,8 +51,8 @@ dependencies:
|
|
52
51
|
version: '0'
|
53
52
|
type: :runtime
|
54
53
|
name: pg
|
55
|
-
- !ruby/object:Gem::Dependency
|
56
54
|
prerelease: false
|
55
|
+
- !ruby/object:Gem::Dependency
|
57
56
|
version_requirements: !ruby/object:Gem::Requirement
|
58
57
|
requirements:
|
59
58
|
- - '>='
|
@@ -66,8 +65,8 @@ dependencies:
|
|
66
65
|
version: '3.0'
|
67
66
|
type: :runtime
|
68
67
|
name: redis
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
68
|
prerelease: false
|
69
|
+
- !ruby/object:Gem::Dependency
|
71
70
|
version_requirements: !ruby/object:Gem::Requirement
|
72
71
|
requirements:
|
73
72
|
- - '>='
|
@@ -80,8 +79,8 @@ dependencies:
|
|
80
79
|
version: '1.3'
|
81
80
|
type: :runtime
|
82
81
|
name: redis-namespace
|
83
|
-
- !ruby/object:Gem::Dependency
|
84
82
|
prerelease: false
|
83
|
+
- !ruby/object:Gem::Dependency
|
85
84
|
version_requirements: !ruby/object:Gem::Requirement
|
86
85
|
requirements:
|
87
86
|
- - '>='
|
@@ -94,8 +93,8 @@ dependencies:
|
|
94
93
|
version: 0.0.1
|
95
94
|
type: :runtime
|
96
95
|
name: callbacks_rb
|
97
|
-
- !ruby/object:Gem::Dependency
|
98
96
|
prerelease: false
|
97
|
+
- !ruby/object:Gem::Dependency
|
99
98
|
version_requirements: !ruby/object:Gem::Requirement
|
100
99
|
requirements:
|
101
100
|
- - '>='
|
@@ -108,8 +107,8 @@ dependencies:
|
|
108
107
|
version: '1.3'
|
109
108
|
type: :runtime
|
110
109
|
name: redis_counters
|
111
|
-
- !ruby/object:Gem::Dependency
|
112
110
|
prerelease: false
|
111
|
+
- !ruby/object:Gem::Dependency
|
113
112
|
version_requirements: !ruby/object:Gem::Requirement
|
114
113
|
requirements:
|
115
114
|
- - '>='
|
@@ -122,8 +121,8 @@ dependencies:
|
|
122
121
|
version: '1.7'
|
123
122
|
type: :development
|
124
123
|
name: bundler
|
125
|
-
- !ruby/object:Gem::Dependency
|
126
124
|
prerelease: false
|
125
|
+
- !ruby/object:Gem::Dependency
|
127
126
|
version_requirements: !ruby/object:Gem::Requirement
|
128
127
|
requirements:
|
129
128
|
- - '>='
|
@@ -136,8 +135,8 @@ dependencies:
|
|
136
135
|
version: '10.0'
|
137
136
|
type: :development
|
138
137
|
name: rake
|
139
|
-
- !ruby/object:Gem::Dependency
|
140
138
|
prerelease: false
|
139
|
+
- !ruby/object:Gem::Dependency
|
141
140
|
version_requirements: !ruby/object:Gem::Requirement
|
142
141
|
requirements:
|
143
142
|
- - '>='
|
@@ -150,8 +149,8 @@ dependencies:
|
|
150
149
|
version: '3.2'
|
151
150
|
type: :development
|
152
151
|
name: rspec
|
153
|
-
- !ruby/object:Gem::Dependency
|
154
152
|
prerelease: false
|
153
|
+
- !ruby/object:Gem::Dependency
|
155
154
|
version_requirements: !ruby/object:Gem::Requirement
|
156
155
|
requirements:
|
157
156
|
- - '>='
|
@@ -164,8 +163,8 @@ dependencies:
|
|
164
163
|
version: '3.2'
|
165
164
|
type: :development
|
166
165
|
name: rspec-rails
|
167
|
-
- !ruby/object:Gem::Dependency
|
168
166
|
prerelease: false
|
167
|
+
- !ruby/object:Gem::Dependency
|
169
168
|
version_requirements: !ruby/object:Gem::Requirement
|
170
169
|
requirements:
|
171
170
|
- - '>='
|
@@ -178,8 +177,8 @@ dependencies:
|
|
178
177
|
version: '3.5'
|
179
178
|
type: :development
|
180
179
|
name: rspec-given
|
181
|
-
- !ruby/object:Gem::Dependency
|
182
180
|
prerelease: false
|
181
|
+
- !ruby/object:Gem::Dependency
|
183
182
|
version_requirements: !ruby/object:Gem::Requirement
|
184
183
|
requirements:
|
185
184
|
- - '>='
|
@@ -192,8 +191,8 @@ dependencies:
|
|
192
191
|
version: 1.0.2
|
193
192
|
type: :development
|
194
193
|
name: appraisal
|
195
|
-
- !ruby/object:Gem::Dependency
|
196
194
|
prerelease: false
|
195
|
+
- !ruby/object:Gem::Dependency
|
197
196
|
version_requirements: !ruby/object:Gem::Requirement
|
198
197
|
requirements:
|
199
198
|
- - '>='
|
@@ -206,8 +205,8 @@ dependencies:
|
|
206
205
|
version: '0'
|
207
206
|
type: :development
|
208
207
|
name: mock_redis
|
209
|
-
- !ruby/object:Gem::Dependency
|
210
208
|
prerelease: false
|
209
|
+
- !ruby/object:Gem::Dependency
|
211
210
|
version_requirements: !ruby/object:Gem::Requirement
|
212
211
|
requirements:
|
213
212
|
- - '>='
|
@@ -220,6 +219,7 @@ dependencies:
|
|
220
219
|
version: '0'
|
221
220
|
type: :development
|
222
221
|
name: apress-changelogger
|
222
|
+
prerelease: false
|
223
223
|
description:
|
224
224
|
email:
|
225
225
|
- bibendi@bk.ru
|
@@ -230,6 +230,7 @@ files:
|
|
230
230
|
- .gitignore
|
231
231
|
- .rspec
|
232
232
|
- Appraisals
|
233
|
+
- CHANGELOG.md
|
233
234
|
- Gemfile
|
234
235
|
- LICENSE.txt
|
235
236
|
- Makefile
|
@@ -250,6 +251,7 @@ files:
|
|
250
251
|
- lib/redis_counters/dumpers/list.rb
|
251
252
|
- lib/redis_counters/dumpers/version.rb
|
252
253
|
- redis_counters-dumpers.gemspec
|
254
|
+
- spec/internal/app/models/stat.rb
|
253
255
|
- spec/internal/app/models/stats_agg_total.rb
|
254
256
|
- spec/internal/app/models/stats_by_day.rb
|
255
257
|
- spec/internal/app/models/stats_total.rb
|
@@ -278,11 +280,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
278
280
|
version: '0'
|
279
281
|
requirements: []
|
280
282
|
rubyforge_project:
|
281
|
-
rubygems_version: 2.
|
283
|
+
rubygems_version: 2.6.1
|
282
284
|
signing_key:
|
283
285
|
specification_version: 4
|
284
286
|
summary: Dump statistics from Redis to DB
|
285
287
|
test_files:
|
288
|
+
- spec/internal/app/models/stat.rb
|
286
289
|
- spec/internal/app/models/stats_agg_total.rb
|
287
290
|
- spec/internal/app/models/stats_by_day.rb
|
288
291
|
- spec/internal/app/models/stats_total.rb
|