redis_counters-dumpers 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b52ecd91c02cece254c010ea63a1e67ed32b9e2
-  data.tar.gz: 24189eaed14d65f10cb264dc6cd01b973c715f68
+  metadata.gz: 9be0aa7c67e74438f7bad1434ffb9f606d81d9f2
+  data.tar.gz: 93c77edc5d86974e29a26607bc3bf086263b0acd
 SHA512:
-  metadata.gz: 46a78d977693f438c9a72b8d018ab3f5e3d36dc51afa0f792b444595c50076881a68351c9172d11a2d1a0fe1f3acfc5b883c190c5496d121b018923c4e497cef
-  data.tar.gz: 1e3c11304907178fbb9f6d3b6d4955a377b74f8cd9c19dc9bba04028f596c2053e7cc40115f28d9d3651eae4cef50c2ba1f4e92ab6cb4a88d02f2a8d7ea5702d
+  metadata.gz: 8e22229c899e2a94b309164023ba10e1d1536037214feb3d5a1ef52cd5b71dcbd585f988bb765f23887f07d5317cb603ee706455c6b585fc90114cd708bfb24e
+  data.tar.gz: d9a35017f37676ffa039f0c9c9f24833e390bae5473454a07dc0d4602886bd890438832193b92eed8697357765c71b440e6aa1bf7617116a3d2bac9dc163c0c8

data/.rspec

@@ -0,0 +1,5 @@
+--color
+--tty
+--format progress
+--order random
+--backtrace

data/Appraisals

@@ -0,0 +1,19 @@
+appraise 'rails3.1' do
+  gem 'activesupport', '~> 3.1.0'
+  gem 'activerecord', '~> 3.1.0'
+end
+
+appraise 'rails3.2' do
+  gem 'activesupport', '~> 3.2.0'
+  gem 'activerecord', '~> 3.2.0'
+end
+
+appraise 'rails4.0' do
+  gem 'activesupport', '~> 4.0.0'
+  gem 'activerecord', '~> 4.0.0'
+end
+
+appraise 'rails4.1' do
+  gem 'activesupport', '~> 4.1.0'
+  gem 'activerecord', '~> 4.1.0'
+end

data/Gemfile

@@ -1,4 +1,9 @@
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in redis_counters-dumpers.gemspec
+
+group :development, :test do
+  gem 'combustion', github: 'pat/combustion', ref: '7d0d24c3f36ce0eb336177fc493be0721bc26665'
+end
+
 gemspec

data/Makefile

@@ -0,0 +1,28 @@
+RAILS_ENV = test
+BUNDLE = RAILS_ENV=${RAILS_ENV} bundle
+BUNDLE_OPTIONS = -j 2
+RSPEC = rspec
+APPRAISAL = appraisal
+
+all: test
+
+test: config/database bundler/install appraisal/install
+	${BUNDLE} exec ${APPRAISAL} ${RSPEC} spec 2>&1
+
+config/database:
+	touch spec/internal/config/database.yml
+	echo 'test:' >> spec/internal/config/database.yml
+	echo '  adapter: postgresql' >> spec/internal/config/database.yml
+	echo '  database: docker' >> spec/internal/config/database.yml
+	echo '  username: docker' >> spec/internal/config/database.yml
+	echo '  host: localhost' >> spec/internal/config/database.yml
+	echo '  min_messages: warning' >> spec/internal/config/database.yml
+
+bundler/install:
+	if ! gem list bundler -i > /dev/null; then \
+	  gem install bundler; \
+	fi
+	${BUNDLE} install ${BUNDLE_OPTIONS}
+
+appraisal/install:
+	${BUNDLE} exec ${APPRAISAL} install

data/gemfiles/.gitignore

@@ -0,0 +1 @@
+*.gemfile.lock

data/gemfiles/rails3.1.gemfile

@@ -0,0 +1,12 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 3.1.0"
+gem "activerecord", "~> 3.1.0"
+
+group :development, :test do
+  gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
+end
+
+gemspec :path => "../"

data/gemfiles/rails3.2.gemfile

@@ -0,0 +1,12 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 3.2.0"
+gem "activerecord", "~> 3.2.0"
+
+group :development, :test do
+  gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
+end
+
+gemspec :path => "../"

data/gemfiles/rails4.0.gemfile

@@ -0,0 +1,12 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 4.0.0"
+gem "activerecord", "~> 4.0.0"
+
+group :development, :test do
+  gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
+end
+
+gemspec :path => "../"

data/gemfiles/rails4.1.gemfile

@@ -0,0 +1,12 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activesupport", "~> 4.1.0"
+gem "activerecord", "~> 4.1.0"
+
+group :development, :test do
+  gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
+end
+
+gemspec :path => "../"

data/lib/redis_counters/dumpers/destination.rb

@@ -1,6 +1,7 @@
 # coding: utf-8
 require 'forwardable'
 require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/object/blank'
 require_relative 'dsl/destination'
 
 module RedisCounters
@@ -48,6 +49,9 @@ module RedisCounters
       # а целевое поле :date, указывает на поле :start_month_date, дампера.
       attr_accessor :fields_map
 
+      # Список полей по которым будет группироваться таблицы с исходными данным, Array
+      attr_accessor :group_by
+
       # Список дополнительных условий, которые применяются при обновлении целевой таблицы, Array of String.
       # Каждое условие представляет собой строку - часть SQL выражения, которое может включать именованные
       # параметры из числа доступных в хеше оббщих параметров дампера: engine.common_params.
@@ -68,7 +72,8 @@ module RedisCounters
             source AS
             (
               SELECT #{selected_fields_expression}
-                FROM #{source_table}
+              FROM #{source_table}
+              #{group_by_expression}
             ),
             updated AS
             (
@@ -83,11 +88,11 @@ module RedisCounters
           INSERT INTO #{target_table} (#{target_fields})
             SELECT #{target_fields}
             FROM source
-          WHERE NOT EXISTS (
-            SELECT 1
+            WHERE NOT EXISTS (
+              SELECT 1
               FROM updated target
-            WHERE #{matching_expression}
-              #{extra_conditions}
+              WHERE #{matching_expression}
+                #{extra_conditions}
           )
         SQL
 
@@ -105,6 +110,11 @@ module RedisCounters
         full_fields_map.map { |target_field, source_field| "#{source_field} as #{target_field}" }.join(', ')
       end
 
+      def group_by_expression
+        return if group_by.blank?
+        'GROUP BY %s' % [group_by.join(', ')]
+      end
+
       def full_fields_map
         fields_map.reverse_merge(Hash[fields.zip(fields)])
       end

data/lib/redis_counters/dumpers/dsl/destination.rb

@@ -17,6 +17,7 @@ module RedisCounters
           varags_setter :fields
           varags_setter :key_fields
           varags_setter :increment_fields
+          varags_setter :group_by
 
           alias_method :take, :fields
 

data/lib/redis_counters/dumpers/engine.rb

@@ -4,6 +4,7 @@ require 'callbacks_rb'
 require 'active_support/core_ext/hash/indifferent_access'
 require 'redis'
 require 'redis/namespace'
+require 'redis_counters'
 require_relative 'dsl/engine'
 
 module RedisCounters
@@ -179,6 +180,8 @@ module RedisCounters
         destinations.each { |dest| dest.merge }
 
         fire_callback(:on_after_merge, self, db_connection)
+
+        drop_temp_table
       end
 
       def fill_temp_table
@@ -245,6 +248,10 @@ module RedisCounters
         SQL
       end
 
+      def drop_temp_table
+        db_connection.execute "DROP TABLE #{temp_table_name}"
+      end
+
       def analyze_table
         db_connection.execute <<-SQL
           ANALYZE #{temp_table_name}

data/lib/redis_counters/dumpers/version.rb

@@ -1,5 +1,5 @@
 module RedisCounters
   module Dumpers
-    VERSION = '0.0.1'
+    VERSION = '0.1.0'
   end
 end

data/lib/redis_counters/dumpers.rb

@@ -1,3 +1,4 @@
+require 'active_record'
 require 'redis_counters/dumpers/version'
 
 module RedisCounters

data/redis_counters-dumpers.gemspec

@@ -19,10 +19,19 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'activesupport', '>= 3.0'
   spec.add_dependency 'activerecord', '>= 3.0'
+  spec.add_dependency 'pg'
   spec.add_dependency 'redis', '>= 3.0'
   spec.add_dependency 'redis-namespace', '>= 1.3'
   spec.add_dependency 'callbacks_rb', '>= 0.0.1'
+  spec.add_dependency 'redis_counters', '>= 1.3'
+
 
   spec.add_development_dependency 'bundler', '>= 1.7'
   spec.add_development_dependency 'rake', '>= 10.0'
+  spec.add_development_dependency 'rspec', '>= 3.2'
+  spec.add_development_dependency 'rspec-rails', '>= 3.2'
+  spec.add_development_dependency 'rspec-given', '>= 3.5'
+  spec.add_development_dependency 'appraisal', '>= 1.0.2'
+  spec.add_development_dependency 'mock_redis'
+  spec.add_development_dependency 'apress-changelogger'
 end

data/spec/internal/app/models/stats_agg_total.rb

@@ -0,0 +1,2 @@
+class StatsAggTotal < ActiveRecord::Base
+end

data/spec/internal/app/models/stats_by_day.rb

@@ -0,0 +1,2 @@
+class StatsByDay < ActiveRecord::Base
+end

data/spec/internal/app/models/stats_total.rb

@@ -0,0 +1,2 @@
+class StatsTotal < ActiveRecord::Base
+end

data/spec/internal/config/.gitignore

@@ -0,0 +1 @@
+database.yml

data/spec/internal/db/schema.rb

@@ -0,0 +1,25 @@
+ActiveRecord::Schema.define do
+  create_table :stats_by_days do |t|
+    t.integer :record_id, null: false
+    t.integer :column_id, null: false
+    t.date :date, null: false
+    t.integer :hits, null: false, default: 0
+  end
+
+  add_index :stats_by_days, [:record_id, :column_id, :date], unique: true
+
+  create_table :stats_totals do |t|
+    t.integer :record_id, null: false
+    t.integer :column_id, null: false
+    t.integer :hits, null: false, default: 0
+  end
+
+  add_index :stats_totals, [:record_id, :column_id], unique: true
+
+  create_table :stats_agg_totals do |t|
+    t.integer :record_id, null: false
+    t.integer :hits, null: false, default: 0
+  end
+
+  add_index :stats_agg_totals, [:record_id], unique: true
+end

data/spec/internal/log/.gitignore

@@ -0,0 +1 @@
+*.log

data/spec/lib/redis_counters/dumpers/engine_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+
+describe RedisCounters::Dumpers::Engine do
+  let(:dumper) do
+    RedisCounters::Dumpers::Engine.build do
+      name :stats_totals
+      fields record_id: :integer,
+             column_id: :integer,
+             value: :integer,
+             date: :date
+
+      destination do
+        model StatsByDay
+        take :record_id, :column_id, :hits, :date
+        key_fields :record_id, :column_id, :date
+        increment_fields :hits
+        map :hits, to: :value
+        condition 'target.date = :date'
+      end
+
+      destination do
+        model StatsTotal
+        take :record_id, :column_id, :hits
+        key_fields :record_id, :column_id
+        increment_fields :hits
+        map :hits, to: :value
+      end
+
+      destination do
+        model StatsAggTotal
+        take :record_id, :hits
+        key_fields :record_id
+        increment_fields :hits
+        map :hits, to: 'sum(value)'
+        group_by :record_id
+      end
+
+      on_before_merge do |dumper, _connection|
+        dumper.common_params = {date: dumper.date.strftime('%Y-%m-%d')}
+      end
+    end
+  end
+
+  let(:prev_date) { Date.new(2015, 1, 19) }
+  let(:prev_date_s) { prev_date.strftime('%Y-%m-%d') }
+
+  let(:date) { Date.new(2015, 1, 20) }
+  let(:date_s) { date.strftime('%Y-%m-%d') }
+
+  let(:counter) do
+    RedisCounters.create_counter(Redis.current,
+      counter_class: RedisCounters::HashCounter,
+      counter_name: :record_hits_by_day,
+      group_keys: [:record_id, :column_id],
+      partition_keys: [:date]
+    )
+  end
+
+  before do
+    allow(dumper).to receive(:redis_session).and_return(MockRedis.new)
+  end
+
+  describe '#process!' do
+    before do
+      counter.increment(date: prev_date_s, record_id: 1, column_id: 100)
+      counter.increment(date: prev_date_s, record_id: 1, column_id: 200)
+      counter.increment(date: prev_date_s, record_id: 1, column_id: 200)
+      counter.increment(date: prev_date_s, record_id: 2, column_id: 100)
+
+      dumper.process!(counter, prev_date)
+
+      counter.increment(date: date_s, record_id: 1, column_id: 100)
+      counter.increment(date: date_s, record_id: 1, column_id: 200)
+      counter.increment(date: date_s, record_id: 1, column_id: 200)
+      counter.increment(date: date_s, record_id: 2, column_id: 100)
+
+      dumper.process!(counter, date)
+    end
+
+    Then { expect(StatsByDay.count).to eq 6 }
+    And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
+    And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: prev_date).first.hits).to eq 2 }
+    And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
+    And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
+    And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: date).first.hits).to eq 2 }
+    And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
+
+    And { expect(StatsTotal.count).to eq 3 }
+    And { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
+    And { expect(StatsTotal.where(record_id: 1, column_id: 200).first.hits).to eq 4 }
+    And { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
+
+    And { expect(StatsAggTotal.count).to eq 2 }
+    And { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 6 }
+    And { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,18 @@
+# coding: utf-8
+require 'bundler/setup'
+require 'redis_counters/dumpers'
+
+require 'combustion'
+Combustion.initialize! :active_record
+
+require 'rspec/rails'
+require 'rspec/given'
+
+require 'mock_redis'
+require 'redis'
+Redis.current = MockRedis.new
+
+RSpec.configure do |config|
+  config.use_transactional_fixtures = true
+  config.before { Redis.current.flushdb }
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis_counters-dumpers
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Merkushin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-29 00:00:00.000000000 Z
+date: 2015-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   prerelease: false
@@ -38,6 +38,20 @@ dependencies:
         version: '3.0'
   type: :runtime
   name: activerecord
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  name: pg
 - !ruby/object:Gem::Dependency
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -80,6 +94,20 @@ dependencies:
         version: 0.0.1
   type: :runtime
   name: callbacks_rb
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  name: redis_counters
 - !ruby/object:Gem::Dependency
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -108,6 +136,90 @@ dependencies:
         version: '10.0'
   type: :development
   name: rake
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  name: rspec
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  name: rspec-rails
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  name: rspec-given
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :development
+  name: appraisal
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  name: mock_redis
+- !ruby/object:Gem::Dependency
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  name: apress-changelogger
 description: 
 email:
 - bibendi@bk.ru
@@ -116,10 +228,18 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rspec
+- Appraisals
 - Gemfile
 - LICENSE.txt
+- Makefile
 - README.md
 - Rakefile
+- gemfiles/.gitignore
+- gemfiles/rails3.1.gemfile
+- gemfiles/rails3.2.gemfile
+- gemfiles/rails4.0.gemfile
+- gemfiles/rails4.1.gemfile
 - lib/redis_counters/dumpers.rb
 - lib/redis_counters/dumpers/destination.rb
 - lib/redis_counters/dumpers/dsl/base.rb
@@ -130,6 +250,14 @@ files:
 - lib/redis_counters/dumpers/list.rb
 - lib/redis_counters/dumpers/version.rb
 - redis_counters-dumpers.gemspec
+- spec/internal/app/models/stats_agg_total.rb
+- spec/internal/app/models/stats_by_day.rb
+- spec/internal/app/models/stats_total.rb
+- spec/internal/config/.gitignore
+- spec/internal/db/schema.rb
+- spec/internal/log/.gitignore
+- spec/lib/redis_counters/dumpers/engine_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/abak-press/redis_counters-dumpers
 licenses:
 - MIT
@@ -154,4 +282,12 @@ rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
 summary: Dump statistics from Redis to DB
-test_files: []
+test_files:
+- spec/internal/app/models/stats_agg_total.rb
+- spec/internal/app/models/stats_by_day.rb
+- spec/internal/app/models/stats_total.rb
+- spec/internal/config/.gitignore
+- spec/internal/db/schema.rb
+- spec/internal/log/.gitignore
+- spec/lib/redis_counters/dumpers/engine_spec.rb
+- spec/spec_helper.rb