redis_counters-dumpers 1.2.1 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8f9e7a48c4a5744051a571af5bda53bcadb83828
-  data.tar.gz: 4218fab18de8290d51d3fc685ba997ac964e6224
+  metadata.gz: e14453e48ca15c05c626c6a9c2b267a50ef1e9da
+  data.tar.gz: '0779a0ddb4eec188a21cf7fe163e565339898583'
 SHA512:
-  metadata.gz: cff6a03f6a5d7f7037e90a5cf953806d05227d247e1bb12261b0dd2d78a10ad59b3e5851f87f61a7e6ebbfb20d4f87d5494f81799804a501996decaa0e5da3fc
-  data.tar.gz: 7454b1bfe68aea37df4cd14aefc65c41ff6586a278738d0da60348e77c60fa51a0e8945ea39ebd794d5e6e1e662bc7b1e64a86db1d7ea28f56f9a9ec10a2824a
+  metadata.gz: e7651d848f01b98b1f5e2c3086f2b46a17b54497fd7692c8f7108bf072fa79f696e498b00ae23c901376430a7d4995157426d2c931af6d922d9388949cc36783
+  data.tar.gz: e972fb15ec3058bb99d53fb6341ae85f5b25f2a301a11257cbf51cfe4ca393fc1030564bf469cf7e05fd6c4680d5ed12049c1f654a146fe10c184d5f3d365f91

data/CHANGELOG.md

@@ -1,7 +1,15 @@
+# v1.2.2
+
+* 2019-02-26 [a57f56a](../../commit/a57f56a) - __(Oleg Perminov)__ Release v1.2.2 
+* 2018-12-21 [6dd9509](../../commit/6dd9509) - __(Oleg Perminov)__ fix: add type text to updating expression 
+https://jira.railsc.ru/browse/CRM-4532
+
+* 2018-12-13 [b197e2a](../../commit/b197e2a) - __(Oleg Perminov)__ feature: add flexibility to persistent types 
+https://jira.railsc.ru/browse/CRM-4532
+
 # v1.2.1
 
 * 2018-12-27 [dca67bc](../../commit/dca67bc) - __(Andrew N. Shalaev)__ feature: add autopublication to rubygems.org 
-* 2018-12-21 [f4e57f5](../../commit/f4e57f5) - __(Andrew N. Shalaev)__ Release v1.2.1 
 * 2018-12-26 [21b57c8](../../commit/21b57c8) - __(Andrew N. Shalaev)__ fix: constraint pg gem 
 * 2018-12-21 [2f4ceab](../../commit/2f4ceab) - __(Andrew N. Shalaev)__ feature: add support for redis >= v4 
 

data/lib/redis_counters/dumpers/destination.rb

@@ -18,6 +18,8 @@ module RedisCounters
       extend Forwardable
       include ::RedisCounters::Dumpers::Dsl::Destination
 
+      VALUE_DELIMITER = ','.freeze
+
       # Ссылка на родительский движек - дампер.
       attr_accessor :engine
 
@@ -76,6 +78,9 @@ module RedisCounters
       # Returns String
       attr_accessor :matching_expr
 
+      # Разделитель значений, String.
+      attr_accessor :value_delimiter
+
       def initialize(engine)
         @engine = engine
         @fields_map = HashWithIndifferentAccess.new
@@ -174,7 +179,16 @@ module RedisCounters
       end
 
       def updating_expression
-        increment_fields.map { |field| "#{field} = COALESCE(target.#{field}, 0) + source.#{field}" }.join(', ')
+        increment_fields.map do |field|
+          case model.columns_hash[field.to_s].type
+          when :datetime, :date
+            "#{field} = source.#{field}"
+          when :text, :string
+            "#{field} = array_to_string(ARRAY[source.#{field}, target.#{field}], '#{delimiter}')"
+          else
+            "#{field} = COALESCE(target.#{field}, 0) + source.#{field}"
+          end
+        end.join(', ')
       end
 
       def matching_expression
@@ -197,6 +211,10 @@ module RedisCounters
 
         "WHERE #{source_conditions.map { |source_condition| "(#{source_condition})" }.join(' AND ')}"
       end
+
+      def delimiter
+        value_delimiter || VALUE_DELIMITER
+      end
     end
   end
 end

data/lib/redis_counters/dumpers/dsl/destination.rb

@@ -13,6 +13,7 @@ module RedisCounters
 
           setter :model
           setter :matching_expr
+          setter :value_delimiter
 
           varags_setter :fields
           varags_setter :key_fields

data/lib/redis_counters/dumpers/version.rb

@@ -1,5 +1,5 @@
 module RedisCounters
   module Dumpers
-    VERSION = '1.2.1'.freeze
+    VERSION = '1.2.2'.freeze
   end
 end

data/redis_counters-dumpers.gemspec

@@ -18,7 +18,6 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'activesupport', '>= 3.0', '< 5'
   spec.add_runtime_dependency 'activerecord', '>= 3.0'
-  spec.add_runtime_dependency 'pg'
   spec.add_runtime_dependency 'redis', '>= 3.0'
   spec.add_runtime_dependency 'redis-namespace', '>= 1.3'
   spec.add_runtime_dependency 'callbacks_rb', '>= 0.0.1'

data/spec/internal/app/models/realtime_stat.rb

@@ -0,0 +1,2 @@
+class RealtimeStat < ActiveRecord::Base
+end

data/spec/internal/db/schema.rb

@@ -52,4 +52,12 @@ ActiveRecord::Schema.define do
     t.integer :value, null: false, default: 0
     t.string :payload
   end
+
+  create_table :realtime_stats do |t|
+    t.integer :record_id, null: false
+    t.integer :column_id, null: false
+    t.integer :hits, null: false, default: 0
+    t.timestamp :date
+    t.string :params
+  end
 end

data/spec/lib/redis_counters/dumpers/engine_spec.rb

@@ -65,100 +65,236 @@ describe RedisCounters::Dumpers::Engine do
 
   describe '#process!' do
     context 'when increment_fields specified' do
-      before do
-        counter.increment(date: prev_date_s, record_id: 1, column_id: 100, subject: '', params: '')
-        counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
-        counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
-        counter.increment(date: prev_date_s, record_id: 2, column_id: 100, subject: nil, params: '')
+      context 'without source conditions' do
+        before do
+          counter.increment(date: prev_date_s, record_id: 1, column_id: 100, subject: '', params: '')
+          counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
+          counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
+          counter.increment(date: prev_date_s, record_id: 2, column_id: 100, subject: nil, params: '')
 
-        params = {a: 1}.stringify_keys.to_s[1..-2]
-        counter.increment(date: prev_date_s, record_id: 3, column_id: 300, subject: nil, params: params)
+          params = {a: 1}.stringify_keys.to_s[1..-2]
+          counter.increment(date: prev_date_s, record_id: 3, column_id: 300, subject: nil, params: params)
 
-        dumper.process!(counter, date: prev_date)
+          dumper.process!(counter, date: prev_date)
 
-        counter.increment(date: date_s, record_id: 1, column_id: 100, subject: '', params: '')
-        counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
-        counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
-        counter.increment(date: date_s, record_id: 2, column_id: 100, subject: nil, params: '')
+          counter.increment(date: date_s, record_id: 1, column_id: 100, subject: '', params: '')
+          counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
+          counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
+          counter.increment(date: date_s, record_id: 2, column_id: 100, subject: nil, params: '')
 
-        dumper.process!(counter, date: date)
-      end
+          dumper.process!(counter, date: date)
+        end
 
-      it { expect(StatsByDay.count).to eq 7 }
-      it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
-      it { expect(StatsByDay.where(record_id: 1, column_id: 200, date: prev_date).first.hits).to eq 2 }
-      it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
-      it { expect(StatsByDay.where(record_id: 3, column_id: 300, date: prev_date).first.params).to eq("a" => "1") }
-      it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
-      it { expect(StatsByDay.where(record_id: 1, column_id: 200, date: date).first.hits).to eq 2 }
-      it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
+        it { expect(StatsByDay.count).to eq 7 }
+        it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
+        it { expect(StatsByDay.where(record_id: 1, column_id: 200, date: prev_date).first.hits).to eq 2 }
+        it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
+        it { expect(StatsByDay.where(record_id: 3, column_id: 300, date: prev_date).first.params).to eq("a" => "1") }
+        it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
+        it { expect(StatsByDay.where(record_id: 1, column_id: 200, date: date).first.hits).to eq 2 }
+        it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
 
-      it { expect(StatsTotal.count).to eq 4 }
-      it { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
-      it { expect(StatsTotal.where(record_id: 1, column_id: 200).first.hits).to eq 4 }
-      it { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
+        it { expect(StatsTotal.count).to eq 4 }
+        it { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
+        it { expect(StatsTotal.where(record_id: 1, column_id: 200).first.hits).to eq 4 }
+        it { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
 
-      it { expect(StatsAggTotal.count).to eq 3 }
-      it { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 6 }
-      it { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
+        it { expect(StatsAggTotal.count).to eq 3 }
+        it { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 6 }
+        it { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
+      end
 
       context 'with source conditions' do
+        context 'when incremented field class is integer' do
+          let(:dumper) do
+            RedisCounters::Dumpers::Engine.build do
+              name :stats_totals
+              fields record_id: :integer,
+                     column_id: :integer,
+                     value: :integer,
+                     date: :date
+
+              destination do
+                model StatsByDay
+                take :record_id, :column_id, :hits, :date
+                key_fields :record_id, :column_id, :date
+                increment_fields :hits
+                map :hits, to: :value
+                condition 'target.date = :date'
+                source_condition 'column_id = 100'
+              end
+
+              destination do
+                model StatsTotal
+                take :record_id, :column_id, :hits
+                key_fields :record_id, :column_id
+                increment_fields :hits
+                map :hits, to: :value
+                source_condition 'column_id = 100'
+              end
+
+              destination do
+                model StatsAggTotal
+                take :record_id, :hits
+                key_fields :record_id
+                increment_fields :hits
+                map :hits, to: 'sum(value)'
+                group_by :record_id
+                source_condition 'column_id = 100'
+              end
+
+              on_before_merge do |dumper, _connection|
+                dumper.common_params = {date: dumper.args[:date].strftime('%Y-%m-%d')}
+              end
+            end
+          end
+
+          before do
+            counter.increment(date: prev_date_s, record_id: 1, column_id: 100, subject: '', params: '')
+            counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
+            counter.increment(date: prev_date_s, record_id: 1, column_id: 200, subject: '', params: '')
+            counter.increment(date: prev_date_s, record_id: 2, column_id: 100, subject: nil, params: '')
+
+            params = {a: 1}.stringify_keys.to_s[1..-2]
+            counter.increment(date: prev_date_s, record_id: 3, column_id: 300, subject: nil, params: params)
+
+            dumper.process!(counter, date: prev_date)
+
+            counter.increment(date: date_s, record_id: 1, column_id: 100, subject: '', params: '')
+            counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
+            counter.increment(date: date_s, record_id: 1, column_id: 200, subject: '', params: '')
+            counter.increment(date: date_s, record_id: 2, column_id: 100, subject: nil, params: '')
+
+            dumper.process!(counter, date: date)
+          end
+
+          it { expect(StatsByDay.count).to eq 4 }
+          it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
+          it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
+          it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
+          it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
+
+          it { expect(StatsTotal.count).to eq 2 }
+          it { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
+          it { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
+
+          it { expect(StatsAggTotal.count).to eq 2 }
+          it { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 2 }
+          it { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
+        end
+      end
+
+      context 'when incremented field class is string' do
         let(:dumper) do
           RedisCounters::Dumpers::Engine.build do
-            name :stats_totals
+            name :realtime_stats
             fields record_id: :integer,
                    column_id: :integer,
                    value: :integer,
-                   date: :date
+                   params: :string,
+                   date: :timestamp
 
             destination do
-              model StatsByDay
-              take :record_id, :column_id, :hits, :date
-              key_fields :record_id, :column_id, :date
-              increment_fields :hits
+              model RealtimeStat
+              take :record_id, :column_id, :date, :hits, :params
+              key_fields :record_id, :column_id
+              increment_fields :hits, :params
+              value_delimiter '; '
               map :hits, to: :value
-              condition 'target.date = :date'
-              source_condition 'column_id = 100'
+              condition 'target.date::date = :date::date'
             end
+          end
+        end
+
+        before do
+          counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: 'abc')
+          dumper.common_params = {date: date, params: 'abc'}
+          dumper.process!(counter, date: date)
+
+          counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: 'xyz')
+          dumper.common_params = {date: date, params: 'xyz'}
+          dumper.process!(counter, date: date)
+        end
+
+        it do
+          expect(RealtimeStat.count).to eq 1
+          expect(RealtimeStat.first.params).to eq 'xyz; abc'
+        end
+      end
+
+      context 'when incremented field class is text' do
+        let(:dumper) do
+          RedisCounters::Dumpers::Engine.build do
+            name :realtime_stats
+            fields record_id: :integer,
+                   column_id: :integer,
+                   value: :integer,
+                   params: :text,
+                   date: :timestamp
 
             destination do
-              model StatsTotal
-              take :record_id, :column_id, :hits
+              model RealtimeStat
+              take :record_id, :column_id, :date, :hits, :params
               key_fields :record_id, :column_id
-              increment_fields :hits
+              increment_fields :hits, :params
               map :hits, to: :value
-              source_condition 'column_id = 100'
+              condition 'target.date::date = :date::date'
             end
+          end
+        end
 
-            destination do
-              model StatsAggTotal
-              take :record_id, :hits
-              key_fields :record_id
-              increment_fields :hits
-              map :hits, to: 'sum(value)'
-              group_by :record_id
-              source_condition 'column_id = 100'
-            end
+        before do
+          counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: 'abc')
+          dumper.common_params = {date: date, params: 'abc'}
+          dumper.process!(counter, date: date)
+
+          counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: 'xyz')
+          dumper.common_params = {date: date, params: 'xyz'}
+          dumper.process!(counter, date: date)
+        end
+
+        it do
+          expect(RealtimeStat.count).to eq 1
+          expect(RealtimeStat.first.params).to eq 'xyz,abc'
+        end
+      end
+
+      context 'when incremented field class is date or time' do
+        let(:current_time) { Date.today.to_time }
+        let(:dumper) do
+          RedisCounters::Dumpers::Engine.build do
+            name :realtime_stats
+            fields record_id: :integer,
+                   column_id: :integer,
+                   value: :integer,
+                   params: :string,
+                   date: :timestamp
 
-            on_before_merge do |dumper, _connection|
-              dumper.common_params = {date: dumper.args[:date].strftime('%Y-%m-%d')}
+            destination do
+              model RealtimeStat
+              take :record_id, :column_id, :hits, :params, :date
+              key_fields :record_id, :column_id
+              increment_fields :hits, :date
+              map :hits, to: :value
+              condition 'target.date::date = :date::date'
             end
           end
         end
 
-        it { expect(StatsByDay.count).to eq 4 }
-        it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
-        it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
-        it { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
-        it { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
+        before do
+          counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: '')
+          dumper.common_params = {date: current_time}
+          dumper.process!(counter, date: date)
 
-        it { expect(StatsTotal.count).to eq 2 }
-        it { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
-        it { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
+          counter.increment(date: date, record_id: 1, column_id: 100, subject: '', params: '')
+          dumper.common_params = {date: current_time}
+          dumper.process!(counter, date: date)
+        end
 
-        it { expect(StatsAggTotal.count).to eq 2 }
-        it { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 2 }
-        it { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
+        it 'update incremented date' do
+          expect(RealtimeStat.count).to eq 1
+          expect(RealtimeStat.first.date).to eq current_time
+        end
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis_counters-dumpers
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.2
 platform: ruby
 authors:
 - Merkushin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-27 00:00:00.000000000 Z
+date: 2019-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -44,20 +44,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: pg
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -270,6 +256,7 @@ files:
 - lib/redis_counters/dumpers/version.rb
 - redis_counters-dumpers.gemspec
 - spec/internal/app/models/nullable_stat.rb
+- spec/internal/app/models/realtime_stat.rb
 - spec/internal/app/models/stat.rb
 - spec/internal/app/models/stats_agg_total.rb
 - spec/internal/app/models/stats_by_day.rb
@@ -304,6 +291,7 @@ specification_version: 4
 summary: Dump statistics from Redis to DB
 test_files:
 - spec/internal/app/models/nullable_stat.rb
+- spec/internal/app/models/realtime_stat.rb
 - spec/internal/app/models/stat.rb
 - spec/internal/app/models/stats_agg_total.rb
 - spec/internal/app/models/stats_by_day.rb