pg_partitioning 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3218f90c14f4626030e7eee68580ba3e8ab6beec
+  data.tar.gz: 6f71ac05ee7bb5fdf82f65335aeba6ab7cc1073a
+SHA512:
+  metadata.gz: 2051b32762a47edc84ce1cf597282dc4700280dd7bb32f747497b6d143a7eefe32f712cb44937dd968bddadfc32e2aec5999d14a05b55c3cf22d1177fa820231
+  data.tar.gz: 0b3f91c0c0f69906e733ac45ce794ddacb773bde5bd0e595f2da720d71168a360f9c8711ff96f28044b11d31e333c8595f0dcd4851052633bf97e2fe66748a73

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2016 Victor M.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,97 @@
+= PgPartitioning
+
+{<img src="https://codeclimate.com/github/victor-magarlamov/pg_partitioning/badges/gpa.svg" />}[https://codeclimate.com/github/victor-magarlamov/pg_partitioning]
+{<img src="https://codeclimate.com/github/victor-magarlamov/pg_partitioning/badges/coverage.svg" />}[https://codeclimate.com/github/victor-magarlamov/pg_partitioning/coverage]
+
+This project rocks and uses MIT-LICENSE.
+
+== Install
+
+Put this line in your Gemfile:
+  gem 'pg_partitioning'
+
+Then bundle:
+  % bundle install
+
+== Usage
+
+  % RAILS_ENV=production rails g partitioning
+
+1) Select the partitioning mode:
+* by single column +value+
+* by dates +template+
+* by +range+
+
+2) Enter +table+ name.
+
+3) Enter +column+ name.
+
+4) Enter condition depending on the selected mode - template +pattern+ or range +step+.
+
+What will happen then...
+
+1) The generator will create two triggers: before insert and after insert. The before insert trigger executes the procedure that created nested table and inserts a new record into it. The after trigger clears the master table.
+
+2) Foreign keys which reference to the master table will delete.
+
+3) Old data will be migrated from master table to child tables.
+
+For more details, see ...
+
+== Examples
+
+Given we have table 'bandits':
+
+| id | name | specialization | born_at
+
+=== Example 1: Partitioning by single column value
+
+  Enter mode: 0
+  Enter table: bandits
+  Enter column: specialization
+
+Create two bandits:
+
+  Bandit.create([{name: 'Al Capone', specialization: 'bootlegger'},
+                 {name: 'Black Bart', specialization: 'robber'}])
+
+Now we have three tables:
+
+* bandits - master - not contains real data
+* bandits_bootlegger - nested - contains only the bootleggers
+* bandits_robber - nested - contains only the robbers
+
+Check this... 
+  
+  SELECT COUNT(*) FROM bandits; (2)
+  SELECT COUNT(*) FROM ONLY bandits; (0)
+  SELECT COUNT(*) FROM ONLY bandits_bootlegger; (1)
+  SELECT COUNT(*) FROM ONLY bandits_robber; (1)
+
+=== Example 2: Partitioning by range
+
+  Enter mode: 1
+  Enter table: bandits
+  Enter column: id
+  Enter step: 10
+
+All bandits with ID from 0 to 9 will be recorded to table 'bandits_0'.
+All bandits with ID from 10 to 19 will be recorded to table 'bandits_1'.
+All bandits with ID from 20 to 29 will be recorded to table 'bandits_2' etc.
+
+=== Example 3: Partitioning by date template
+
+  Enter mode: 2
+  Enter table: bandits
+  Enter column: born_at
+  Enter pattern: YYYYMM
+
+Create two bandits:
+
+  Bandit.create([{name: 'Al Capone', born_at: '1899-01-17'},
+                 {name: 'Charles Luciano', born_at: '1897-11-24'}])
+
+And now we have two child tables:
+
+* bandits_1899_01
+* bandits_1897_11

data/Rakefile

@@ -0,0 +1,23 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'PgPartitioning'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+
+
+
+Bundler::GemHelper.install_tasks
+

data/config/locales/en.yml

@@ -0,0 +1,28 @@
+en:
+  pg_partitioning:
+    enter: "Enter the %{quest}" 
+    quests:
+      mode: "type of the partitioning - equal(0), step(1), date(2): "
+      table_name: "name of the table: "
+      column_name: "name of the column: "
+      cond_step: "step (e.g. 1000): "
+      cond_date: "pattern (e.g. YYYY_MM or MM): "
+    failure:
+      answer: "I cannot accept your answer!"
+      mode: "Invalid value."
+      step: "Invalid value. Should be eat then 0"
+      pattern: "Invalid value."
+      no_table: "Table with some column is not exists!"
+      column_type: "Invalid column type."
+      blank_cond: "Value is not set."
+      other: "An error has occurred."
+    progress:
+      insert_master: "Insert master func: %{state}"
+      before_insert_trigger: "Before Insert trigger: %{state}"
+      drop_master: "Drop master func: %{state}"
+      after_insert_trigger: "After Insert trigger: %{state}"
+      drop_fk: "Delete foreign keys: %{state}"
+      migration: "Migrate old data: %{state}"
+      finish: "Done!"
+    messages:
+      partition_mode: "Recommend to set constraint_exclusion in 'partition' state for best perfomance (your current state is %{current})."

data/lib/generators/partitioning_generator.rb

@@ -0,0 +1,18 @@
+require 'pg_partitioning/input_master'
+require 'pg_partitioning/partitioning_master'
+
+class PartitioningGenerator < Rails::Generators::Base
+  
+  def partitioning
+    input_master = PgPartitioning::InputMaster.new
+    input_master.intro
+    
+    mode        = input_master.ask_mode.to_i
+    table_name  = input_master.ask_table_name
+    column_name = input_master.ask_column_name
+    cond        = input_master.ask_cond(PgPartitioning::PartitioningMaster::MODES[mode]) 
+    
+    master = PgPartitioning::PartitioningMaster.new(table_name, column_name, mode, cond)
+    master.partitioning
+  end
+end

data/lib/pg_partitioning/input_master.rb

@@ -0,0 +1,83 @@
+require 'pg_partitioning/printer'
+
+module PgPartitioning
+  class InputMaster
+    include Printer
+    
+    %w(mode table_name column_name cond).each do |act|
+      define_method "ask_#{act}" do |argument = nil|
+        begin
+          quest_key = argument.blank? ? act : "#{act}_#{argument}"
+          quest_val = I18n.t "pg_partitioning.quests.#{quest_key}", raise: true
+          res = ask quest_val
+          if send "#{quest_key}_valid?", res
+            return res
+          else
+            alert @error_message
+            send "ask_#{act}", argument unless ENV['RAILS_ENV'] == 'test'
+          end
+        rescue I18n::MissingTranslationData
+          return nil
+        end
+      end
+    end
+
+    def method_missing(name, *args)
+      valid?(*args) if name.to_s.include?('_valid?')
+    end
+    
+    def intro
+      info <<-INTRO
+      Hi! It's time to do partitioning)
+      I can do it in different ways:
+      
+      by EQUALity
+          - partition will be created by value of column,
+          - in this case you should only set column name;
+      
+      by STEP
+          - partition will be created by value of column divided by the step and rounded,
+          - in this case you should else set number of the step,
+          - a type of the column should be a numeric;
+      
+      by DATE
+          - partition will be created by pattern from date,
+          - in this case you should else set pattern for parsing the date,
+          - acceptable (itself or in combination): Y, YY, YYY, YYYY, MM, D, DD, DDD, W, WW, HH24 
+          - a type of the column should be a date/timestamp.
+      INTRO
+    end
+
+    private
+      def ask(text)
+        text_color WHITE
+        print I18n.t('pg_partitioning.enter', quest: text)
+        gets.chomp
+      end
+
+      def valid?(text)
+        res = !text.blank?
+        @error_message = I18n.t 'pg_partitioning.failure.answer' unless res
+        res
+      end
+
+      def mode_valid?(text)
+        res = %w(0 1 2).include? text
+        @error_message = I18n.t 'pg_partitioning.failure.mode' unless res
+        res
+      end
+      
+      def cond_step_valid?(text)
+        res = text.to_i > 0
+        @error_message = I18n.t "pg_partitioning.failure.step" unless res
+        res
+      end
+      
+      def cond_date_valid?(text)
+        res = /Y{1,4}|M{2}|D{1,3}|W{1,2}|HH24/ =~ text
+        @error_message = I18n.t "pg_partitioning.failure.pattern" unless res
+        res
+      end
+  end
+end
+

data/lib/pg_partitioning/partitioning_master.rb

@@ -0,0 +1,78 @@
+require 'pg_partitioning/printer'
+require 'pg_partitioning/strategies/equal'
+require 'pg_partitioning/strategies/date'
+require 'pg_partitioning/strategies/step'
+
+module PgPartitioning
+  class PartitioningMaster
+    include Printer
+    
+    MODES = %w(equal step date).freeze
+
+    def initialize(table_name, column_name, mode, cond=nil)
+      @table_name = table_name 
+      @column_name = column_name
+      @cond = cond
+      @sql = ActiveRecord::Base.connection()
+      
+      klass = "PgPartitioning::Strategies::#{MODES[mode].classify}"
+      @strategy = klass.constantize.new(@table_name, @column_name, @cond, @sql)
+    end
+
+    def partitioning
+      @strategy.partitioning!
+      
+      drop_foreign_keys
+      migration
+      
+      mode = show_value_of('constraint_exclusion')
+      if mode != 'partition'
+        alert I18n.t('pg_partitioning.messages.partition_mode', current: mode)
+      end
+      
+      info I18n.t 'pg_partitioning.progress.finish'
+    rescue => e
+      alert e.message || I18n.t('pg_partitioning.failure.other')
+    end
+
+    private
+      def drop_foreign_keys
+        fk_info = []
+        query = ActiveRecord::Base.send(
+                  :sanitize_sql_array,
+                  ["SELECT tc.constraint_name, tc.table_name, kcu.column_name, 
+                           ccu.table_name AS foreign_table_name,
+                           ccu.column_name AS foreign_column_name 
+                    FROM   information_schema.table_constraints AS tc 
+                    JOIN   information_schema.key_column_usage AS kcu ON tc.constraint_name = kcu.constraint_name
+                    JOIN   information_schema.constraint_column_usage AS ccu ON ccu.constraint_name = tc.constraint_name
+                    WHERE  constraint_type = 'FOREIGN KEY' AND ccu.table_name=?;", @table_name])
+           
+        @sql.execute(query).each do |r|
+          fk_info << r
+        end
+
+        fk_info.each do |r|
+          @sql.execute "ALTER TABLE #{r['table_name']} DROP CONSTRAINT #{r['constraint_name']};"
+        end
+        
+        info I18n.t('pg_partitioning.progress.drop_fk', state: 'OK')
+      end
+
+      def migration
+        @sql.execute <<-SQL
+          CREATE MATERIALIZED VIEW temp_table AS SELECT * FROM #{@table_name};
+          DELETE FROM #{@table_name};
+          INSERT INTO #{@table_name} (SELECT * FROM temp_table);
+          DROP MATERIALIZED VIEW temp_table;
+        SQL
+        info I18n.t('pg_partitioning.progress.migration', state: 'OK')
+      end
+      
+      def show_value_of(param)
+        res = ''
+        @sql.execute("SHOW #{param};").each{ |r| res = r[param] }
+        res
+      end
+  end
+end

data/lib/pg_partitioning/printer.rb

@@ -0,0 +1,28 @@
+module PgPartitioning
+  module Printer
+    COLORS = [GREEN="\e[32m", WHITE="\e[0m", RED="\e[31m"]
+  
+    def text_color(color)
+      print color
+    end
+
+    def print_row(mes, color = WHITE)
+      text_color color
+      print mes + "\n"
+      text_color WHITE
+    end
+    
+    def info(mes)
+      print_row mes, GREEN
+    end
+    
+    def alert(mes)
+      print_row mes, RED
+    end
+    
+    def message(mes)
+      print_row mes
+    end
+  end
+end
+

data/lib/pg_partitioning/strategies/base.rb

@@ -0,0 +1,72 @@
+require 'pg_partitioning/printer'
+
+module PgPartitioning
+  module Strategies
+    class Base
+      include Printer
+
+      def initialize(table, column, cond, sql_conn)
+        @table_name = table
+        @column_name = column
+        @cond = cond
+        @sql = sql_conn
+      end
+        
+      def partitioning!
+        raise error_message unless valid?
+        create_insert_master_function
+        create_trigger('insert_master', 'before')
+        create_drop_function
+        create_trigger('delete_master', 'after')
+      end
+      
+      protected
+
+      def valid?
+        @data_type = column_data_type
+        @error_message = I18n.t "pg_partitioning.failure.no_table" if @data_type.blank?
+        @error_message.blank?
+      end
+      
+      def error_message
+        @error_message || I18n.t("pg_partitioning.failure.other")
+      end
+
+      def column_data_type
+        res = nil
+        query = ActiveRecord::Base.send(:sanitize_sql_array,
+                                        ["SELECT data_type FROM information_schema.columns
+                                          WHERE table_name = ? AND column_name = ?;",
+                                         @table_name, @column_name])
+        @sql.execute(query).each do |r|
+          res = r['data_type']
+        end
+        res
+      end
+      
+      def create_trigger(master, mode)
+        drop_trigger(mode)
+        @sql.execute "CREATE TRIGGER #{@table_name}_#{mode}_insert_trigger
+                      #{mode} INSERT ON #{@table_name}
+                      FOR EACH ROW EXECUTE PROCEDURE #{@table_name}_#{master}();"
+        info I18n.t("pg_partitioning.progress.#{mode}_insert_trigger", state: "OK")
+      end
+
+      def drop_trigger(mode)
+        @sql.execute "DROP TRIGGER IF EXISTS #{@table_name}_#{mode}_insert_trigger ON #{@table_name};"
+      end
+
+      def create_drop_function
+        @sql.execute "CREATE OR REPLACE FUNCTION #{@table_name}_delete_master() RETURNS TRIGGER AS $$
+                      DECLARE
+                        row #{@table_name.to_sym}%rowtype;
+                      BEGIN
+                        DELETE FROM ONLY #{@table_name} WHERE id = NEW.id RETURNING * INTO row;
+                        RETURN row;
+                      END;
+                      $$ LANGUAGE plpgsql;"
+        info I18n.t("pg_partitioning.progress.drop_master", state: "OK")
+      end
+    end
+  end
+end