redis_counters-dumpers 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +31 -0
- data/Rakefile +2 -0
- data/lib/redis_counters/dumpers.rb +9 -0
- data/lib/redis_counters/dumpers/destination.rb +128 -0
- data/lib/redis_counters/dumpers/dsl/base.rb +46 -0
- data/lib/redis_counters/dumpers/dsl/destination.rb +42 -0
- data/lib/redis_counters/dumpers/dsl/engine.rb +39 -0
- data/lib/redis_counters/dumpers/dsl/list.rb +27 -0
- data/lib/redis_counters/dumpers/engine.rb +284 -0
- data/lib/redis_counters/dumpers/list.rb +17 -0
- data/lib/redis_counters/dumpers/version.rb +5 -0
- data/redis_counters-dumpers.gemspec +28 -0
- metadata +157 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA1:
|
|
3
|
+
metadata.gz: 0b52ecd91c02cece254c010ea63a1e67ed32b9e2
|
|
4
|
+
data.tar.gz: 24189eaed14d65f10cb264dc6cd01b973c715f68
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 46a78d977693f438c9a72b8d018ab3f5e3d36dc51afa0f792b444595c50076881a68351c9172d11a2d1a0fe1f3acfc5b883c190c5496d121b018923c4e497cef
|
|
7
|
+
data.tar.gz: 1e3c11304907178fbb9f6d3b6d4955a377b74f8cd9c19dc9bba04028f596c2053e7cc40115f28d9d3651eae4cef50c2ba1f4e92ab6cb4a88d02f2a8d7ea5702d
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
Copyright (c) 2015 bibendi
|
|
2
|
+
|
|
3
|
+
MIT License
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
|
6
|
+
a copy of this software and associated documentation files (the
|
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
|
11
|
+
the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be
|
|
14
|
+
included in all copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# RedisCounters::Dumpers
|
|
2
|
+
|
|
3
|
+
Dump statistics from Redis to DB
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
Add this line to your application's Gemfile:
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
gem 'redis_counters-dumpers'
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
And then execute:
|
|
14
|
+
|
|
15
|
+
$ bundle
|
|
16
|
+
|
|
17
|
+
Or install it yourself as:
|
|
18
|
+
|
|
19
|
+
$ gem install redis_counters-dumpers
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
22
|
+
|
|
23
|
+
TODO: Write usage instructions here
|
|
24
|
+
|
|
25
|
+
## Contributing
|
|
26
|
+
|
|
27
|
+
1. Fork it ( https://github.com/abak-press/redis_counters-dumpers/fork )
|
|
28
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
|
29
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
|
30
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
|
31
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
require 'forwardable'
|
|
3
|
+
require 'active_support/core_ext/hash/indifferent_access'
|
|
4
|
+
require_relative 'dsl/destination'
|
|
5
|
+
|
|
6
|
+
module RedisCounters
|
|
7
|
+
module Dumpers
|
|
8
|
+
# Класс представляет конечную точку сохранения данных счетчика.
|
|
9
|
+
#
|
|
10
|
+
# Описывает в какую модель (таблицу), какие поля имеющиеся в распоряжении дампера,
|
|
11
|
+
# должны быть сохранены и каким образом.
|
|
12
|
+
#
|
|
13
|
+
# По сути, мерджит указанные поля из temp - таблицы, дампера
|
|
14
|
+
# в указанную таблицу.
|
|
15
|
+
#
|
|
16
|
+
# Может использоваться как напрямую так и с помощью DSL (см. модуль RedisCounters::Dumpers::Dsl::Destination).
|
|
17
|
+
class Destination
|
|
18
|
+
extend Forwardable
|
|
19
|
+
include ::RedisCounters::Dumpers::Dsl::Destination
|
|
20
|
+
|
|
21
|
+
# Ссылка на родительский движек - дампер.
|
|
22
|
+
attr_accessor :engine
|
|
23
|
+
|
|
24
|
+
# Модель, в таблицу, которой будет производится мердж данных, AR::Model.
|
|
25
|
+
attr_accessor :model
|
|
26
|
+
|
|
27
|
+
# Список полей, из доступных дамперу, которые необходимо сохранить, Array.
|
|
28
|
+
attr_accessor :fields
|
|
29
|
+
|
|
30
|
+
# Список полей, по комбинации которых, будет происходить определение существования записи,
|
|
31
|
+
# при мердже данных, Array.
|
|
32
|
+
attr_accessor :key_fields
|
|
33
|
+
|
|
34
|
+
# Список полей, которые будет инкрементированы при обновлении существующей записи, Array.
|
|
35
|
+
attr_accessor :increment_fields
|
|
36
|
+
|
|
37
|
+
# Карта полей - карта псевдонимов полей, Hash.
|
|
38
|
+
# Названия полей в целевой таблице, могут отличаться от названий полей дампера.
|
|
39
|
+
# Для сопоставления полей целевой таблицы и дампера, необходимо заполнить карту соответствия.
|
|
40
|
+
# Карта, заполняется только для тех полей, названия которых отличаются.
|
|
41
|
+
# Во всех свойствах, содержащий указания полей: fields, key_fields, increment_fields, conditions
|
|
42
|
+
# используются имена конечных полей целевой таблицы.
|
|
43
|
+
#
|
|
44
|
+
# Example:
|
|
45
|
+
# fields_map = {:pages => :value, :date => :start_month_date}
|
|
46
|
+
#
|
|
47
|
+
# Означает, что целевое поле :pages, указывает на поле :value, дампера,
|
|
48
|
+
# а целевое поле :date, указывает на поле :start_month_date, дампера.
|
|
49
|
+
attr_accessor :fields_map
|
|
50
|
+
|
|
51
|
+
# Список дополнительных условий, которые применяются при обновлении целевой таблицы, Array of String.
|
|
52
|
+
# Каждое условие представляет собой строку - часть SQL выражения, которое может включать именованные
|
|
53
|
+
# параметры из числа доступных в хеше оббщих параметров дампера: engine.common_params.
|
|
54
|
+
# Условия соеденяются через AND.
|
|
55
|
+
attr_accessor :conditions
|
|
56
|
+
|
|
57
|
+
def initialize(engine)
|
|
58
|
+
@engine = engine
|
|
59
|
+
@fields_map = HashWithIndifferentAccess.new
|
|
60
|
+
@conditions = []
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def merge
|
|
64
|
+
target_fields = fields.join(', ')
|
|
65
|
+
|
|
66
|
+
sql = <<-SQL
|
|
67
|
+
WITH
|
|
68
|
+
source AS
|
|
69
|
+
(
|
|
70
|
+
SELECT #{selected_fields_expression}
|
|
71
|
+
FROM #{source_table}
|
|
72
|
+
),
|
|
73
|
+
updated AS
|
|
74
|
+
(
|
|
75
|
+
UPDATE #{target_table} target
|
|
76
|
+
SET
|
|
77
|
+
#{updating_expression}
|
|
78
|
+
FROM source
|
|
79
|
+
WHERE #{matching_expression}
|
|
80
|
+
#{extra_conditions}
|
|
81
|
+
RETURNING target.*
|
|
82
|
+
)
|
|
83
|
+
INSERT INTO #{target_table} (#{target_fields})
|
|
84
|
+
SELECT #{target_fields}
|
|
85
|
+
FROM source
|
|
86
|
+
WHERE NOT EXISTS (
|
|
87
|
+
SELECT 1
|
|
88
|
+
FROM updated target
|
|
89
|
+
WHERE #{matching_expression}
|
|
90
|
+
#{extra_conditions}
|
|
91
|
+
)
|
|
92
|
+
SQL
|
|
93
|
+
|
|
94
|
+
sql = model.send(:sanitize_sql, [sql, engine.common_params])
|
|
95
|
+
connection.execute sql
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def_delegator :model, :connection
|
|
99
|
+
def_delegator :model, :quoted_table_name, :target_table
|
|
100
|
+
def_delegator :engine, :temp_table_name, :source_table
|
|
101
|
+
|
|
102
|
+
protected
|
|
103
|
+
|
|
104
|
+
def selected_fields_expression
|
|
105
|
+
full_fields_map.map { |target_field, source_field| "#{source_field} as #{target_field}" }.join(', ')
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def full_fields_map
|
|
109
|
+
fields_map.reverse_merge(Hash[fields.zip(fields)])
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def updating_expression
|
|
113
|
+
increment_fields.map { |field| "#{field} = COALESCE(target.#{field}, 0) + source.#{field}" }.join(', ')
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def matching_expression
|
|
117
|
+
source_key_fields = key_fields.map { |field| "source.#{field}" }.join(', ')
|
|
118
|
+
target_key_fields = key_fields.map { |field| "target.#{field}" }.join(', ')
|
|
119
|
+
"(#{source_key_fields}) = (#{target_key_fields})"
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def extra_conditions
|
|
123
|
+
result = conditions.map { |condition| "(#{condition})" }.join(' AND ')
|
|
124
|
+
result.present? ? "AND #{result}" : result
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
module RedisCounters
|
|
3
|
+
module Dumpers
|
|
4
|
+
module Dsl
|
|
5
|
+
# Базовый класс для создания DSL к другим классам.
|
|
6
|
+
# Класс обертка, который имеет все свойства, включая callbacks,
|
|
7
|
+
# который просто настраивает целевой класс через его стандартные свойства.
|
|
8
|
+
# Профит в простоте реализации DSL, в его изоляции от основного класса,
|
|
9
|
+
# в разделении логики основного класса и DSL к нему.
|
|
10
|
+
class Base
|
|
11
|
+
attr_accessor :target
|
|
12
|
+
|
|
13
|
+
class << self
|
|
14
|
+
def setter(*method_names)
|
|
15
|
+
method_names.each do |name|
|
|
16
|
+
send :define_method, name do |data|
|
|
17
|
+
target.send "#{name}=".to_sym, data
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def varags_setter(*method_names)
|
|
23
|
+
method_names.each do |name|
|
|
24
|
+
send :define_method, name do |*data|
|
|
25
|
+
target.send "#{name}=".to_sym, data.flatten
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def callback_setter(*method_names)
|
|
31
|
+
method_names.each do |name|
|
|
32
|
+
send :define_method, name do |method = nil, &block|
|
|
33
|
+
target.send "#{name}=".to_sym, method, &block
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def initialize(target, &block)
|
|
40
|
+
@target = target
|
|
41
|
+
instance_eval(&block)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
require 'active_support/concern'
|
|
3
|
+
require_relative 'base'
|
|
4
|
+
|
|
5
|
+
module RedisCounters
|
|
6
|
+
module Dumpers
|
|
7
|
+
module Dsl
|
|
8
|
+
# Модуль реализующий DSL для класса Destination
|
|
9
|
+
module Destination
|
|
10
|
+
extend ActiveSupport::Concern
|
|
11
|
+
|
|
12
|
+
class Configuration < ::RedisCounters::Dumpers::Dsl::Base
|
|
13
|
+
alias_method :destination, :target
|
|
14
|
+
|
|
15
|
+
setter :model
|
|
16
|
+
|
|
17
|
+
varags_setter :fields
|
|
18
|
+
varags_setter :key_fields
|
|
19
|
+
varags_setter :increment_fields
|
|
20
|
+
|
|
21
|
+
alias_method :take, :fields
|
|
22
|
+
|
|
23
|
+
def map(field, target_field)
|
|
24
|
+
destination.fields_map.merge!(field.to_sym => target_field[:to])
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def condition(value)
|
|
28
|
+
destination.conditions << value
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
module ClassMethods
|
|
33
|
+
def build(engine, &block)
|
|
34
|
+
destination = new(engine)
|
|
35
|
+
Configuration.new(destination, &block)
|
|
36
|
+
destination
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
require 'active_support/concern'
|
|
3
|
+
require_relative 'base'
|
|
4
|
+
|
|
5
|
+
module RedisCounters
|
|
6
|
+
module Dumpers
|
|
7
|
+
module Dsl
|
|
8
|
+
# Модуль реализующий DSL для класса Engine::Base
|
|
9
|
+
module Engine
|
|
10
|
+
extend ActiveSupport::Concern
|
|
11
|
+
|
|
12
|
+
class Configuration < ::RedisCounters::Dumpers::Dsl::Base
|
|
13
|
+
alias_method :engine, :target
|
|
14
|
+
|
|
15
|
+
setter :name
|
|
16
|
+
setter :fields
|
|
17
|
+
setter :temp_table_name
|
|
18
|
+
|
|
19
|
+
callback_setter :on_before_merge
|
|
20
|
+
callback_setter :on_prepare_row
|
|
21
|
+
callback_setter :on_after_merge
|
|
22
|
+
callback_setter :on_after_delete
|
|
23
|
+
|
|
24
|
+
def destination(&block)
|
|
25
|
+
engine.destinations << ::RedisCounters::Dumpers::Destination.build(engine, &block)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
module ClassMethods
|
|
30
|
+
def build(&block)
|
|
31
|
+
engine = new
|
|
32
|
+
Configuration.new(engine, &block)
|
|
33
|
+
engine
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
require 'active_support/concern'
|
|
3
|
+
require 'redis_counters/dumpers/engine'
|
|
4
|
+
|
|
5
|
+
module RedisCounters
|
|
6
|
+
module Dumpers
|
|
7
|
+
module Dsl
|
|
8
|
+
module List
|
|
9
|
+
extend ActiveSupport::Concern
|
|
10
|
+
|
|
11
|
+
module ClassMethods
|
|
12
|
+
def build(&block)
|
|
13
|
+
instance = new
|
|
14
|
+
instance.instance_eval(&block)
|
|
15
|
+
instance
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def dumper(id, &block)
|
|
20
|
+
engine = ::RedisCounters::Dumpers::Engine.build(&block)
|
|
21
|
+
engine.name = id
|
|
22
|
+
@dumpers[id] = engine
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
require 'forwardable'
|
|
3
|
+
require 'callbacks_rb'
|
|
4
|
+
require 'active_support/core_ext/hash/indifferent_access'
|
|
5
|
+
require 'redis'
|
|
6
|
+
require 'redis/namespace'
|
|
7
|
+
require_relative 'dsl/engine'
|
|
8
|
+
|
|
9
|
+
module RedisCounters
|
|
10
|
+
module Dumpers
|
|
11
|
+
# Класс дампер (движек дампера) - класс осуществляющий перенос данных счетчика в БД.
|
|
12
|
+
#
|
|
13
|
+
# Может использоваться как напрямую так и с помощью DSL (см. модуль RedisCounters::Dumpers::Dsl::Engine).
|
|
14
|
+
#
|
|
15
|
+
# Общий алгоритм работы:
|
|
16
|
+
# - копируем данные счетчика в временную таблицу
|
|
17
|
+
# - мерджим данные во все целевые таблицы
|
|
18
|
+
# - удаляем перенесенные данные из счетчика
|
|
19
|
+
#
|
|
20
|
+
# Все destinations должны быть в рамках одной БД.
|
|
21
|
+
# Все действия происходят в рамках соединения БД, первой destination.
|
|
22
|
+
#
|
|
23
|
+
# Example:
|
|
24
|
+
# dumper = Dumper.build do
|
|
25
|
+
# name :hits_by_day
|
|
26
|
+
#
|
|
27
|
+
# fields => {
|
|
28
|
+
# :company_id => :integer,
|
|
29
|
+
# :value => :integer,
|
|
30
|
+
# :date => :date,
|
|
31
|
+
# :start_month_date => :date,
|
|
32
|
+
# }
|
|
33
|
+
#
|
|
34
|
+
# destination do
|
|
35
|
+
# model CompanyStatisticTotalByDay
|
|
36
|
+
# take :company_id, :pages, :date
|
|
37
|
+
# key_fields :company_id, :date
|
|
38
|
+
# increment_fields :pages
|
|
39
|
+
# map :pages, :to => :value
|
|
40
|
+
# condition 'target.date = :date'
|
|
41
|
+
# end
|
|
42
|
+
#
|
|
43
|
+
# destination do
|
|
44
|
+
# model CompanyStatisticTotalByMonth
|
|
45
|
+
# take :company_id, :pages, :date
|
|
46
|
+
# key_fields :company_id, :date
|
|
47
|
+
# increment_fields :pages
|
|
48
|
+
# map :pages, :to => :value
|
|
49
|
+
# map :date, :to => :start_month_date
|
|
50
|
+
# condition 'target.date = :start_month_date'
|
|
51
|
+
# end
|
|
52
|
+
#
|
|
53
|
+
# on_before_merge do |dumper, connection|
|
|
54
|
+
# dumper.common_params = {
|
|
55
|
+
# :date => dumper.date.strftime('%Y-%m-%d'),
|
|
56
|
+
# :start_month_date => dumper.date.beginning_of_month.strftime('%Y-%m-%d'),
|
|
57
|
+
# }
|
|
58
|
+
# end
|
|
59
|
+
# end
|
|
60
|
+
#
|
|
61
|
+
# dumper.process!(counter, Date.yesterday)
|
|
62
|
+
#
|
|
63
|
+
# В результате все данные счетчика за вчера, будут
|
|
64
|
+
# смерджены в целевые таблицы, по ключевым полям: company_id и date,
|
|
65
|
+
# причем все поля кроме pages, будут просто записаны в таблицы,
|
|
66
|
+
# а поле pages будет инкрементировано с текущим значением, при обновлении.
|
|
67
|
+
# Данные будут удалены из счетчика.
|
|
68
|
+
# Все действия производятся транзакционно, как в БД, так и в Redis.
|
|
69
|
+
class Engine
|
|
70
|
+
include CallbacksRb
|
|
71
|
+
include ::RedisCounters::Dumpers::Dsl::Engine
|
|
72
|
+
extend Forwardable
|
|
73
|
+
|
|
74
|
+
DATE_FORMAT = '%Y-%m-%d'.freeze
|
|
75
|
+
|
|
76
|
+
# properties/accessors
|
|
77
|
+
|
|
78
|
+
# Название дампера
|
|
79
|
+
attr_reader :name
|
|
80
|
+
|
|
81
|
+
# Список доступных для сохранение в целевые таблицы полей и их типов данных, в виде Hash.
|
|
82
|
+
# Доступны следующие типы данных: string, integer, date, timestamp, boolean.
|
|
83
|
+
# Преобразование типов производится непосредственно перед мерджем в целевые таблицы.
|
|
84
|
+
#
|
|
85
|
+
# Example:
|
|
86
|
+
# fields = {:company_id => :integer, :date => :timestamp}
|
|
87
|
+
attr_reader :fields
|
|
88
|
+
|
|
89
|
+
# Массив, целевых моделей для сохранение данных, Array.
|
|
90
|
+
# Каждый элемент массива это экземпляр класса Engine::Destination.
|
|
91
|
+
attr_accessor :destinations
|
|
92
|
+
|
|
93
|
+
# Название temp таблицы, используемой для переноса данных.
|
|
94
|
+
# По умолчанию: "tmp_#{dumper_name}"
|
|
95
|
+
attr_accessor :temp_table_name
|
|
96
|
+
|
|
97
|
+
# Хеш общий параметров.
|
|
98
|
+
# Данные хеш мерджится в каждую, поступающую от счетчика, строку данных.
|
|
99
|
+
attr_accessor :common_params
|
|
100
|
+
|
|
101
|
+
attr_reader :counter
|
|
102
|
+
attr_reader :date
|
|
103
|
+
|
|
104
|
+
# callbacks
|
|
105
|
+
|
|
106
|
+
# Вызывается, перед процессом мерджа данных, в рамках БД - транзакции.
|
|
107
|
+
# Параметры: dumper, db_connection.
|
|
108
|
+
callback :on_before_merge
|
|
109
|
+
|
|
110
|
+
# Вызывается для каждой строки, полученной от счетчика.
|
|
111
|
+
# Позволяет вычисляить дополнительные данные, которые необходимы для сохранения,
|
|
112
|
+
# или произвести предварительную обработку данных от счетчика перед сохранением.
|
|
113
|
+
# В in/out параметре row передается строка данных от счетчика, в виде хеша.
|
|
114
|
+
# Так же, row содержит все общие данные, заданные для дампера в свойстве common_params.
|
|
115
|
+
# Параметры: dumper, row.
|
|
116
|
+
callback :on_prepare_row
|
|
117
|
+
|
|
118
|
+
# Вызывается, по окончанию процесса мерджа данных, в рамках БД - транзакции.
|
|
119
|
+
# Параметры: dumper, db_connection.
|
|
120
|
+
callback :on_after_merge
|
|
121
|
+
|
|
122
|
+
# Вызывается, по окончанию процесса удаления данных из счетчика, в рамках redis - транзакции.
|
|
123
|
+
# Параметры: dumper, redis_connection.
|
|
124
|
+
callback :on_after_delete
|
|
125
|
+
|
|
126
|
+
# Public: Производит перенос данных счетчика.
|
|
127
|
+
#
|
|
128
|
+
# counter - экземпляр счетчика.
|
|
129
|
+
# date - Date - дата, за которую производится перенос данных.
|
|
130
|
+
#
|
|
131
|
+
# Returns Fixnum - кол-во обработанных строк.
|
|
132
|
+
#
|
|
133
|
+
def process!(counter, date)
|
|
134
|
+
@counter, @date = counter, date
|
|
135
|
+
|
|
136
|
+
db_transaction do
|
|
137
|
+
merge_data
|
|
138
|
+
start_redis_transaction
|
|
139
|
+
delete_from_redis
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
commit_redis_transaction
|
|
143
|
+
|
|
144
|
+
rows_processed
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def initialize
|
|
148
|
+
@destinations = []
|
|
149
|
+
@common_params = {}
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def fields=(value)
|
|
153
|
+
@fields = value.with_indifferent_access
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def name=(value)
|
|
157
|
+
@name = value
|
|
158
|
+
@temp_table_name = "tmp_#{@name}"
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
protected
|
|
162
|
+
|
|
163
|
+
attr_accessor :rows_processed
|
|
164
|
+
|
|
165
|
+
def_delegator :redis_session, :multi, :start_redis_transaction
|
|
166
|
+
def_delegator :redis_session, :exec, :commit_redis_transaction
|
|
167
|
+
def_delegator :db_connection, :transaction, :db_transaction
|
|
168
|
+
def_delegator :db_connection, :quote
|
|
169
|
+
|
|
170
|
+
def merge_data
|
|
171
|
+
fire_callback(:on_before_merge, self, db_connection)
|
|
172
|
+
|
|
173
|
+
# копируем данные счетчика в временную таблицу
|
|
174
|
+
create_temp_table
|
|
175
|
+
fill_temp_table
|
|
176
|
+
analyze_table
|
|
177
|
+
|
|
178
|
+
# мерджим в целевые таблицы
|
|
179
|
+
destinations.each { |dest| dest.merge }
|
|
180
|
+
|
|
181
|
+
fire_callback(:on_after_merge, self, db_connection)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def fill_temp_table
|
|
185
|
+
@rows_processed = counter.data(:date => formatted_date) do |batch|
|
|
186
|
+
@current_batch = batch
|
|
187
|
+
prepare_batch
|
|
188
|
+
insert_batch
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def prepare_batch
|
|
193
|
+
fields_keys = fields.keys
|
|
194
|
+
|
|
195
|
+
@current_batch.map! do |row|
|
|
196
|
+
row.merge!(common_params)
|
|
197
|
+
fire_callback(:on_prepare_row, self, row)
|
|
198
|
+
|
|
199
|
+
# выбираем из хеша только указанные поля
|
|
200
|
+
fields_keys.inject(HashWithIndifferentAccess.new) do |result, (field)|
|
|
201
|
+
result.merge!(field => row.fetch(field))
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def insert_batch
|
|
207
|
+
db_connection.execute <<-SQL
|
|
208
|
+
INSERT INTO #{temp_table_name} VALUES #{batch_data}
|
|
209
|
+
SQL
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def batch_data
|
|
213
|
+
@current_batch.map! do |row|
|
|
214
|
+
values = row.map do |field, value|
|
|
215
|
+
next 'null' if value.nil?
|
|
216
|
+
fields.fetch(field).eql?(:integer) ? value : quote(value)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
"(#{values.join(',')})"
|
|
220
|
+
end.join(',')
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def delete_from_redis
|
|
224
|
+
redis_session.pipelined do |redis|
|
|
225
|
+
counter.partitions(:date => formatted_date).each do |partition|
|
|
226
|
+
counter.delete_partition_direct!(partition, redis)
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
fire_callback(:on_after_delete, self, redis_session)
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def redis_session
|
|
234
|
+
@redis_session ||= begin
|
|
235
|
+
redis = ::Redis.new(counter.redis.client.options)
|
|
236
|
+
::Redis::Namespace.new(counter.redis.namespace, :redis => redis)
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def create_temp_table
|
|
241
|
+
db_connection.execute <<-SQL
|
|
242
|
+
CREATE TEMP TABLE #{temp_table_name} (
|
|
243
|
+
#{columns_definition}
|
|
244
|
+
) ON COMMIT DROP
|
|
245
|
+
SQL
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def analyze_table
|
|
249
|
+
db_connection.execute <<-SQL
|
|
250
|
+
ANALYZE #{temp_table_name}
|
|
251
|
+
SQL
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def columns_definition
|
|
255
|
+
@fields.map do |field, type|
|
|
256
|
+
pg_field_type = case type
|
|
257
|
+
when :string, :text
|
|
258
|
+
'character varying(4000)'
|
|
259
|
+
when :integer, :serial, :number
|
|
260
|
+
'integer'
|
|
261
|
+
when :date
|
|
262
|
+
'date'
|
|
263
|
+
when :timestamp
|
|
264
|
+
'timestamp'
|
|
265
|
+
when :boolean
|
|
266
|
+
'boolean'
|
|
267
|
+
else
|
|
268
|
+
raise 'Unknown datatype %s for %s field' % [type, field]
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
"#{field} #{pg_field_type}"
|
|
272
|
+
end.join(',')
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def formatted_date
|
|
276
|
+
date.strftime(DATE_FORMAT)
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
def db_connection
|
|
280
|
+
destinations.first.connection
|
|
281
|
+
end
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
require 'active_support/core_ext/hash/indifferent_access'
|
|
3
|
+
require_relative 'dsl/list'
|
|
4
|
+
|
|
5
|
+
module RedisCounters
|
|
6
|
+
module Dumpers
|
|
7
|
+
class List
|
|
8
|
+
include ::RedisCounters::Dumpers::Dsl::List
|
|
9
|
+
|
|
10
|
+
attr_accessor :dumpers
|
|
11
|
+
|
|
12
|
+
def initialize
|
|
13
|
+
@dumpers = HashWithIndifferentAccess.new
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
|
+
require 'redis_counters/dumpers/version'
|
|
5
|
+
|
|
6
|
+
Gem::Specification.new do |spec|
|
|
7
|
+
spec.name = 'redis_counters-dumpers'
|
|
8
|
+
spec.version = RedisCounters::Dumpers::VERSION
|
|
9
|
+
spec.authors = ['Merkushin']
|
|
10
|
+
spec.email = ['bibendi@bk.ru']
|
|
11
|
+
spec.summary = 'Dump statistics from Redis to DB'
|
|
12
|
+
spec.homepage = 'https://github.com/abak-press/redis_counters-dumpers'
|
|
13
|
+
spec.license = 'MIT'
|
|
14
|
+
|
|
15
|
+
spec.files = `git ls-files -z`.split("\x0")
|
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
|
18
|
+
spec.require_paths = ['lib']
|
|
19
|
+
|
|
20
|
+
spec.add_dependency 'activesupport', '>= 3.0'
|
|
21
|
+
spec.add_dependency 'activerecord', '>= 3.0'
|
|
22
|
+
spec.add_dependency 'redis', '>= 3.0'
|
|
23
|
+
spec.add_dependency 'redis-namespace', '>= 1.3'
|
|
24
|
+
spec.add_dependency 'callbacks_rb', '>= 0.0.1'
|
|
25
|
+
|
|
26
|
+
spec.add_development_dependency 'bundler', '>= 1.7'
|
|
27
|
+
spec.add_development_dependency 'rake', '>= 10.0'
|
|
28
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: redis_counters-dumpers
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.0.1
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Merkushin
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2015-01-29 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
prerelease: false
|
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - '>='
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '3.0'
|
|
20
|
+
requirement: !ruby/object:Gem::Requirement
|
|
21
|
+
requirements:
|
|
22
|
+
- - '>='
|
|
23
|
+
- !ruby/object:Gem::Version
|
|
24
|
+
version: '3.0'
|
|
25
|
+
type: :runtime
|
|
26
|
+
name: activesupport
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
prerelease: false
|
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - '>='
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '3.0'
|
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
|
35
|
+
requirements:
|
|
36
|
+
- - '>='
|
|
37
|
+
- !ruby/object:Gem::Version
|
|
38
|
+
version: '3.0'
|
|
39
|
+
type: :runtime
|
|
40
|
+
name: activerecord
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
prerelease: false
|
|
43
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - '>='
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '3.0'
|
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
|
49
|
+
requirements:
|
|
50
|
+
- - '>='
|
|
51
|
+
- !ruby/object:Gem::Version
|
|
52
|
+
version: '3.0'
|
|
53
|
+
type: :runtime
|
|
54
|
+
name: redis
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
prerelease: false
|
|
57
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - '>='
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: '1.3'
|
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
|
63
|
+
requirements:
|
|
64
|
+
- - '>='
|
|
65
|
+
- !ruby/object:Gem::Version
|
|
66
|
+
version: '1.3'
|
|
67
|
+
type: :runtime
|
|
68
|
+
name: redis-namespace
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
prerelease: false
|
|
71
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - '>='
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: 0.0.1
|
|
76
|
+
requirement: !ruby/object:Gem::Requirement
|
|
77
|
+
requirements:
|
|
78
|
+
- - '>='
|
|
79
|
+
- !ruby/object:Gem::Version
|
|
80
|
+
version: 0.0.1
|
|
81
|
+
type: :runtime
|
|
82
|
+
name: callbacks_rb
|
|
83
|
+
- !ruby/object:Gem::Dependency
|
|
84
|
+
prerelease: false
|
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - '>='
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: '1.7'
|
|
90
|
+
requirement: !ruby/object:Gem::Requirement
|
|
91
|
+
requirements:
|
|
92
|
+
- - '>='
|
|
93
|
+
- !ruby/object:Gem::Version
|
|
94
|
+
version: '1.7'
|
|
95
|
+
type: :development
|
|
96
|
+
name: bundler
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
prerelease: false
|
|
99
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - '>='
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '10.0'
|
|
104
|
+
requirement: !ruby/object:Gem::Requirement
|
|
105
|
+
requirements:
|
|
106
|
+
- - '>='
|
|
107
|
+
- !ruby/object:Gem::Version
|
|
108
|
+
version: '10.0'
|
|
109
|
+
type: :development
|
|
110
|
+
name: rake
|
|
111
|
+
description:
|
|
112
|
+
email:
|
|
113
|
+
- bibendi@bk.ru
|
|
114
|
+
executables: []
|
|
115
|
+
extensions: []
|
|
116
|
+
extra_rdoc_files: []
|
|
117
|
+
files:
|
|
118
|
+
- .gitignore
|
|
119
|
+
- Gemfile
|
|
120
|
+
- LICENSE.txt
|
|
121
|
+
- README.md
|
|
122
|
+
- Rakefile
|
|
123
|
+
- lib/redis_counters/dumpers.rb
|
|
124
|
+
- lib/redis_counters/dumpers/destination.rb
|
|
125
|
+
- lib/redis_counters/dumpers/dsl/base.rb
|
|
126
|
+
- lib/redis_counters/dumpers/dsl/destination.rb
|
|
127
|
+
- lib/redis_counters/dumpers/dsl/engine.rb
|
|
128
|
+
- lib/redis_counters/dumpers/dsl/list.rb
|
|
129
|
+
- lib/redis_counters/dumpers/engine.rb
|
|
130
|
+
- lib/redis_counters/dumpers/list.rb
|
|
131
|
+
- lib/redis_counters/dumpers/version.rb
|
|
132
|
+
- redis_counters-dumpers.gemspec
|
|
133
|
+
homepage: https://github.com/abak-press/redis_counters-dumpers
|
|
134
|
+
licenses:
|
|
135
|
+
- MIT
|
|
136
|
+
metadata: {}
|
|
137
|
+
post_install_message:
|
|
138
|
+
rdoc_options: []
|
|
139
|
+
require_paths:
|
|
140
|
+
- lib
|
|
141
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
142
|
+
requirements:
|
|
143
|
+
- - '>='
|
|
144
|
+
- !ruby/object:Gem::Version
|
|
145
|
+
version: '0'
|
|
146
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
147
|
+
requirements:
|
|
148
|
+
- - '>='
|
|
149
|
+
- !ruby/object:Gem::Version
|
|
150
|
+
version: '0'
|
|
151
|
+
requirements: []
|
|
152
|
+
rubyforge_project:
|
|
153
|
+
rubygems_version: 2.4.2
|
|
154
|
+
signing_key:
|
|
155
|
+
specification_version: 4
|
|
156
|
+
summary: Dump statistics from Redis to DB
|
|
157
|
+
test_files: []
|