redis_counters-dumpers 1.2.1 → 1.2.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 8f9e7a48c4a5744051a571af5bda53bcadb83828
4
- data.tar.gz: 4218fab18de8290d51d3fc685ba997ac964e6224
3
+ metadata.gz: e14453e48ca15c05c626c6a9c2b267a50ef1e9da
4
+ data.tar.gz: '0779a0ddb4eec188a21cf7fe163e565339898583'
5
5
  SHA512:
6
- metadata.gz: cff6a03f6a5d7f7037e90a5cf953806d05227d247e1bb12261b0dd2d78a10ad59b3e5851f87f61a7e6ebbfb20d4f87d5494f81799804a501996decaa0e5da3fc
7
- data.tar.gz: 7454b1bfe68aea37df4cd14aefc65c41ff6586a278738d0da60348e77c60fa51a0e8945ea39ebd794d5e6e1e662bc7b1e64a86db1d7ea28f56f9a9ec10a2824a
6
+ metadata.gz: e7651d848f01b98b1f5e2c3086f2b46a17b54497fd7692c8f7108bf072fa79f696e498b00ae23c901376430a7d4995157426d2c931af6d922d9388949cc36783
7
+ data.tar.gz: e972fb15ec3058bb99d53fb6341ae85f5b25f2a301a11257cbf51cfe4ca393fc1030564bf469cf7e05fd6c4680d5ed12049c1f654a146fe10c184d5f3d365f91
@@ -1,7 +1,15 @@
1
+ # v1.2.2
2
+
3
+ * 2019-02-26 [a57f56a](../../commit/a57f56a) - __(Oleg Perminov)__ Release v1.2.2
4
+ * 2018-12-21 [6dd9509](../../commit/6dd9509) - __(Oleg Perminov)__ fix: add type text to updating expression
5
+ https://jira.railsc.ru/browse/CRM-4532
6
+
7
+ * 2018-12-13 [b197e2a](../../commit/b197e2a) - __(Oleg Perminov)__ feature: add flexibility to persistent types
8
+ https://jira.railsc.ru/browse/CRM-4532
9
+
1
10
  # v1.2.1
2
11
 
3
12
  * 2018-12-27 [dca67bc](../../commit/dca67bc) - __(Andrew N. Shalaev)__ feature: add autopublication to rubygems.org
4
- * 2018-12-21 [f4e57f5](../../commit/f4e57f5) - __(Andrew N. Shalaev)__ Release v1.2.1
5
13
  * 2018-12-26 [21b57c8](../../commit/21b57c8) - __(Andrew N. Shalaev)__ fix: constraint pg gem
6
14
  * 2018-12-21 [2f4ceab](../../commit/2f4ceab) - __(Andrew N. Shalaev)__ feature: add support for redis >= v4
7
15
 
@@ -18,6 +18,8 @@ module RedisCounters
18
18
  extend Forwardable
19
19
  include ::RedisCounters::Dumpers::Dsl::Destination
20
20
 
21
+ VALUE_DELIMITER = ','.freeze
22
+
21
23
  # Ссылка на родительский движек - дампер.
22
24
  attr_accessor :engine
23
25
 
@@ -76,6 +78,9 @@ module RedisCounters
76
78
  # Returns String
77
79
  attr_accessor :matching_expr
78
80
 
81
+ # Разделитель значений, String.
82
+ attr_accessor :value_delimiter
83
+
79
84
  def initialize(engine)
80
85
  @engine = engine
81
86
  @fields_map = HashWithIndifferentAccess.new
@@ -174,7 +179,16 @@ module RedisCounters
174
179
  end
175
180
 
176
181
  def updating_expression
177
- increment_fields.map { |field| "#{field} = COALESCE(target.#{field}, 0) + source.#{field}" }.join(', ')
182
+ increment_fields.map do |field|
183
+ case model.columns_hash[field.to_s].type
184
+ when :datetime, :date
185
+ "#{field} = source.#{field}"
186
+ when :text, :string
187
+ "#{field} = array_to_string(ARRAY[source.#{field}, target.#{field}], '#{delimiter}')"
188
+ else
189
+ "#{field} = COALESCE(target.#{field}, 0) + source.#{field}"
190
+ end
191
+ end.join(', ')
178
192
  end
179
193
 
180
194
  def matching_expression
@@ -197,6 +211,10 @@ module RedisCounters
197
211
 
198
212
  "WHERE #{source_conditions.map { |source_condition| "(#{source_condition})" }.join(' AND ')}"
199
213
  end
214
+
215
+ def delimiter
216
+ value_delimiter || VALUE_DELIMITER
217
+ end
200
218
  end
201
219
  end
202
220
  end
@@ -13,6 +13,7 @@ module RedisCounters
13
13
 
14
14
  setter :model
15
15
  setter :matching_expr
16
+ setter :value_delimiter
16
17
 
17
18
  varags_setter :fields
18
19
  varags_setter :key_fields
@@ -1,5 +1,5 @@
1
1
  module RedisCounters
2
2
  module Dumpers
3
- VERSION = '1.2.1'.freeze
3
+ VERSION = '1.2.2'.freeze
4
4
  end
5
5
  end
@@ -18,7 +18,6 @@ Gem::Specification.new do |spec|
18
18
 
19
19
  spec.add_runtime_dependency 'activesupport', '>= 3.0', '< 5'
20
20
  spec.add_runtime_dependency 'activerecord', '>= 3.0'
21
- spec.add_runtime_dependency 'pg'
22
21
  spec.add_runtime_dependency 'redis', '>= 3.0'
23
22
  spec.add_runtime_dependency 'redis-namespace', '>= 1.3'
24
23
  spec.add_runtime_dependency 'callbacks_rb', '>= 0.0.1'
@@ -0,0 +1,2 @@
1
+ class RealtimeStat < ActiveRecord::Base
2
+ end
@@ -52,4 +52,12 @@ ActiveRecord::Schema.define do
52
52
  t.integer :value, null: false, default: 0
53
53
  t.string :payload
54
54
  end
55
+
56
+ create_table :realtime_stats do |t|
57
+ t.integer :record_id, null: false
58
+ t.integer :column_id, null: false
59
+ t.integer :hits, null: false, default: 0
60
+ t.timestamp :date
61
+ t.string :params
62
+ end
55
63
  end
@@ -65,100 +65,236 @@ describe RedisCounters::Dumpers::Engine do
65
65
 
66
66
  describe '#process!' do
67
67
  context 'when increment_fields specified' do
68
- before do
69
- counter.increment(date: prev_date_s, record_id: 1, column_id: 100, subject: '', params: '')
70
- counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
71
- counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
72
- counter.increment(date: prev_date_s, record_id: 2, column_id: 100, subject: nil, params: '')
68
+ context 'without source conditions' do
69
+ before do
70
+ counter.increment(date: prev_date_s, record_id: 1, column_id: 100, subject: '', params: '')
71
+ counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
72
+ counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
73
+ counter.increment(date: prev_date_s, record_id: 2, column_id: 100, subject: nil, params: '')
73
74
 
74
- params = {a: 1}.stringify_keys.to_s[1..-2]
75
- counter.increment(date: prev_date_s, record_id: 3, column_id: 300, subject: nil, params: params)
75
+ params = {a: 1}.stringify_keys.to_s[1..-2]
76
+ counter.increment(date: prev_date_s, record_id: 3, column_id: 300, subject: nil, params: params)
76
77
 
77
- dumper.process!(counter, date: prev_date)
78
+ dumper.process!(counter, date: prev_date)
78
79
 
79
- counter.increment(date: date_s, record_id: 1, column_id: 100, subject: '', params: '')
80
- counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
81
- counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
82
- counter.increment(date: date_s, record_id: 2, column_id: 100, subject: nil, params: '')
80
+ counter.increment(date: date_s, record_id: 1, column_id: 100, subject: '', params: '')
81
+ counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
82
+ counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
83
+ counter.increment(date: date_s, record_id: 2, column_id: 100, subject: nil, params: '')
83
84
 
84
- dumper.process!(counter, date: date)
85
- end
85
+ dumper.process!(counter, date: date)
86
+ end
86
87
 
87
- it { expect(StatsByDay.count).to eq 7 }
88
- it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
89
- it { expect(StatsByDay.where(record_id: 1, column_id: 200, date: prev_date).first.hits).to eq 2 }
90
- it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
91
- it { expect(StatsByDay.where(record_id: 3, column_id: 300, date: prev_date).first.params).to eq("a" => "1") }
92
- it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
93
- it { expect(StatsByDay.where(record_id: 1, column_id: 200, date: date).first.hits).to eq 2 }
94
- it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
88
+ it { expect(StatsByDay.count).to eq 7 }
89
+ it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
90
+ it { expect(StatsByDay.where(record_id: 1, column_id: 200, date: prev_date).first.hits).to eq 2 }
91
+ it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
92
+ it { expect(StatsByDay.where(record_id: 3, column_id: 300, date: prev_date).first.params).to eq("a" => "1") }
93
+ it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
94
+ it { expect(StatsByDay.where(record_id: 1, column_id: 200, date: date).first.hits).to eq 2 }
95
+ it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
95
96
 
96
- it { expect(StatsTotal.count).to eq 4 }
97
- it { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
98
- it { expect(StatsTotal.where(record_id: 1, column_id: 200).first.hits).to eq 4 }
99
- it { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
97
+ it { expect(StatsTotal.count).to eq 4 }
98
+ it { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
99
+ it { expect(StatsTotal.where(record_id: 1, column_id: 200).first.hits).to eq 4 }
100
+ it { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
100
101
 
101
- it { expect(StatsAggTotal.count).to eq 3 }
102
- it { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 6 }
103
- it { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
102
+ it { expect(StatsAggTotal.count).to eq 3 }
103
+ it { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 6 }
104
+ it { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
105
+ end
104
106
 
105
107
  context 'with source conditions' do
108
+ context 'when incremented field class is integer' do
109
+ let(:dumper) do
110
+ RedisCounters::Dumpers::Engine.build do
111
+ name :stats_totals
112
+ fields record_id: :integer,
113
+ column_id: :integer,
114
+ value: :integer,
115
+ date: :date
116
+
117
+ destination do
118
+ model StatsByDay
119
+ take :record_id, :column_id, :hits, :date
120
+ key_fields :record_id, :column_id, :date
121
+ increment_fields :hits
122
+ map :hits, to: :value
123
+ condition 'target.date = :date'
124
+ source_condition 'column_id = 100'
125
+ end
126
+
127
+ destination do
128
+ model StatsTotal
129
+ take :record_id, :column_id, :hits
130
+ key_fields :record_id, :column_id
131
+ increment_fields :hits
132
+ map :hits, to: :value
133
+ source_condition 'column_id = 100'
134
+ end
135
+
136
+ destination do
137
+ model StatsAggTotal
138
+ take :record_id, :hits
139
+ key_fields :record_id
140
+ increment_fields :hits
141
+ map :hits, to: 'sum(value)'
142
+ group_by :record_id
143
+ source_condition 'column_id = 100'
144
+ end
145
+
146
+ on_before_merge do |dumper, _connection|
147
+ dumper.common_params = {date: dumper.args[:date].strftime('%Y-%m-%d')}
148
+ end
149
+ end
150
+ end
151
+
152
+ before do
153
+ counter.increment(date: prev_date_s, record_id: 1, column_id: 100, subject: '', params: '')
154
+ counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
155
+ counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
156
+ counter.increment(date: prev_date_s, record_id: 2, column_id: 100, subject: nil, params: '')
157
+
158
+ params = {a: 1}.stringify_keys.to_s[1..-2]
159
+ counter.increment(date: prev_date_s, record_id: 3, column_id: 300, subject: nil, params: params)
160
+
161
+ dumper.process!(counter, date: prev_date)
162
+
163
+ counter.increment(date: date_s, record_id: 1, column_id: 100, subject: '', params: '')
164
+ counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
165
+ counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
166
+ counter.increment(date: date_s, record_id: 2, column_id: 100, subject: nil, params: '')
167
+
168
+ dumper.process!(counter, date: date)
169
+ end
170
+
171
+ it { expect(StatsByDay.count).to eq 4 }
172
+ it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
173
+ it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
174
+ it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
175
+ it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
176
+
177
+ it { expect(StatsTotal.count).to eq 2 }
178
+ it { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
179
+ it { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
180
+
181
+ it { expect(StatsAggTotal.count).to eq 2 }
182
+ it { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 2 }
183
+ it { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
184
+ end
185
+ end
186
+
187
+ context 'when incremented field class is string' do
106
188
  let(:dumper) do
107
189
  RedisCounters::Dumpers::Engine.build do
108
- name :stats_totals
190
+ name :realtime_stats
109
191
  fields record_id: :integer,
110
192
  column_id: :integer,
111
193
  value: :integer,
112
- date: :date
194
+ params: :string,
195
+ date: :timestamp
113
196
 
114
197
  destination do
115
- model StatsByDay
116
- take :record_id, :column_id, :hits, :date
117
- key_fields :record_id, :column_id, :date
118
- increment_fields :hits
198
+ model RealtimeStat
199
+ take :record_id, :column_id, :date, :hits, :params
200
+ key_fields :record_id, :column_id
201
+ increment_fields :hits, :params
202
+ value_delimiter '; '
119
203
  map :hits, to: :value
120
- condition 'target.date = :date'
121
- source_condition 'column_id = 100'
204
+ condition 'target.date::date = :date::date'
122
205
  end
206
+ end
207
+ end
208
+
209
+ before do
210
+ counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: 'abc')
211
+ dumper.common_params = {date: date, params: 'abc'}
212
+ dumper.process!(counter, date: date)
213
+
214
+ counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: 'xyz')
215
+ dumper.common_params = {date: date, params: 'xyz'}
216
+ dumper.process!(counter, date: date)
217
+ end
218
+
219
+ it do
220
+ expect(RealtimeStat.count).to eq 1
221
+ expect(RealtimeStat.first.params).to eq 'xyz; abc'
222
+ end
223
+ end
224
+
225
+ context 'when incremented field class is text' do
226
+ let(:dumper) do
227
+ RedisCounters::Dumpers::Engine.build do
228
+ name :realtime_stats
229
+ fields record_id: :integer,
230
+ column_id: :integer,
231
+ value: :integer,
232
+ params: :text,
233
+ date: :timestamp
123
234
 
124
235
  destination do
125
- model StatsTotal
126
- take :record_id, :column_id, :hits
236
+ model RealtimeStat
237
+ take :record_id, :column_id, :date, :hits, :params
127
238
  key_fields :record_id, :column_id
128
- increment_fields :hits
239
+ increment_fields :hits, :params
129
240
  map :hits, to: :value
130
- source_condition 'column_id = 100'
241
+ condition 'target.date::date = :date::date'
131
242
  end
243
+ end
244
+ end
132
245
 
133
- destination do
134
- model StatsAggTotal
135
- take :record_id, :hits
136
- key_fields :record_id
137
- increment_fields :hits
138
- map :hits, to: 'sum(value)'
139
- group_by :record_id
140
- source_condition 'column_id = 100'
141
- end
246
+ before do
247
+ counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: 'abc')
248
+ dumper.common_params = {date: date, params: 'abc'}
249
+ dumper.process!(counter, date: date)
250
+
251
+ counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: 'xyz')
252
+ dumper.common_params = {date: date, params: 'xyz'}
253
+ dumper.process!(counter, date: date)
254
+ end
255
+
256
+ it do
257
+ expect(RealtimeStat.count).to eq 1
258
+ expect(RealtimeStat.first.params).to eq 'xyz,abc'
259
+ end
260
+ end
261
+
262
+ context 'when incremented field class is date or time' do
263
+ let(:current_time) { Date.today.to_time }
264
+ let(:dumper) do
265
+ RedisCounters::Dumpers::Engine.build do
266
+ name :realtime_stats
267
+ fields record_id: :integer,
268
+ column_id: :integer,
269
+ value: :integer,
270
+ params: :string,
271
+ date: :timestamp
142
272
 
143
- on_before_merge do |dumper, _connection|
144
- dumper.common_params = {date: dumper.args[:date].strftime('%Y-%m-%d')}
273
+ destination do
274
+ model RealtimeStat
275
+ take :record_id, :column_id, :hits, :params, :date
276
+ key_fields :record_id, :column_id
277
+ increment_fields :hits, :date
278
+ map :hits, to: :value
279
+ condition 'target.date::date = :date::date'
145
280
  end
146
281
  end
147
282
  end
148
283
 
149
- it { expect(StatsByDay.count).to eq 4 }
150
- it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
151
- it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
152
- it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
153
- it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
284
+ before do
285
+ counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: '')
286
+ dumper.common_params = {date: current_time}
287
+ dumper.process!(counter, date: date)
154
288
 
155
- it { expect(StatsTotal.count).to eq 2 }
156
- it { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
157
- it { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
289
+ counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: '')
290
+ dumper.common_params = {date: current_time}
291
+ dumper.process!(counter, date: date)
292
+ end
158
293
 
159
- it { expect(StatsAggTotal.count).to eq 2 }
160
- it { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 2 }
161
- it { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
294
+ it 'update incremented date' do
295
+ expect(RealtimeStat.count).to eq 1
296
+ expect(RealtimeStat.first.date).to eq current_time
297
+ end
162
298
  end
163
299
  end
164
300
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: redis_counters-dumpers
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.1
4
+ version: 1.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Merkushin
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-12-27 00:00:00.000000000 Z
11
+ date: 2019-03-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -44,20 +44,6 @@ dependencies:
44
44
  - - ">="
45
45
  - !ruby/object:Gem::Version
46
46
  version: '3.0'
47
- - !ruby/object:Gem::Dependency
48
- name: pg
49
- requirement: !ruby/object:Gem::Requirement
50
- requirements:
51
- - - ">="
52
- - !ruby/object:Gem::Version
53
- version: '0'
54
- type: :runtime
55
- prerelease: false
56
- version_requirements: !ruby/object:Gem::Requirement
57
- requirements:
58
- - - ">="
59
- - !ruby/object:Gem::Version
60
- version: '0'
61
47
  - !ruby/object:Gem::Dependency
62
48
  name: redis
63
49
  requirement: !ruby/object:Gem::Requirement
@@ -270,6 +256,7 @@ files:
270
256
  - lib/redis_counters/dumpers/version.rb
271
257
  - redis_counters-dumpers.gemspec
272
258
  - spec/internal/app/models/nullable_stat.rb
259
+ - spec/internal/app/models/realtime_stat.rb
273
260
  - spec/internal/app/models/stat.rb
274
261
  - spec/internal/app/models/stats_agg_total.rb
275
262
  - spec/internal/app/models/stats_by_day.rb
@@ -304,6 +291,7 @@ specification_version: 4
304
291
  summary: Dump statistics from Redis to DB
305
292
  test_files:
306
293
  - spec/internal/app/models/nullable_stat.rb
294
+ - spec/internal/app/models/realtime_stat.rb
307
295
  - spec/internal/app/models/stat.rb
308
296
  - spec/internal/app/models/stats_agg_total.rb
309
297
  - spec/internal/app/models/stats_by_day.rb