pg_partitions 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5fc6179ef26414c9abbf9d233068064d2c309f30
+  data.tar.gz: e3ea73ed26170bfa49a7343633b1720beeeadcab
+SHA512:
+  metadata.gz: 46dfeb813305c00f1a07e05f7073c85ffcd092954193b2a9744b7570c2c18df23b8fb1d9bc7df73b74cb051da41bd0bf924583c53f60d648c25ac6803db3f3f2
+  data.tar.gz: 86c772af301a20e45e9b7b2a4cf14a65bff62d3ccf4fb5a60b314f24c9ddc3a79c72eda275128cc809fc483aabb1bb487ea3f4289ad23e2a4b271d18e60596e4

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.3
+before_install: gem install bundler -v 1.14.4

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pg_partitions.gemspec
+gemspec
+gem 'pry'

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Ray Zane
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,112 @@
+# PgPartitions
+
+Partitioning postgres takes some doing. PgPartitions adds methods to your migrations to help you manage them.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pg_partitions'
+```
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+Imagine you have a comments table with millions of rows and your queries are starting to be a bit slow. Postgres partitioning allows yo to divide your comments table into smaller tables.
+
+In a migration, you'll first need to include `PgPartitions`.
+
+```ruby
+class PartitionComments < ActiveRecord::Migration[5.1]
+  include PgPartitions
+
+  def change
+    # ...
+  end
+end
+```
+
+Let's assume we have a column called year that stores the year the comment was created. We can partition our table based on the value of that column:
+
+```ruby
+add_partition :comments, :comments_2016, check: 'year = 2016'
+add_partition :comments, :comments_2017, check: 'year = 2017'
+```
+
+After we create our partitions, the query plan is going to change a little bit:
+
+```ruby
+Comment.all.explain
+=> EXPLAIN for: SELECT "comments".* FROM "comments"
+                               QUERY PLAN
+------------------------------------------------------------------------
+ Append  (cost=0.00..60.80 rows=4081 width=12)
+   ->  Seq Scan on comments  (cost=0.00..0.00 rows=1 width=12)
+   ->  Seq Scan on comments_2016  (cost=0.00..30.40 rows=2040 width=12)
+   ->  Seq Scan on comments_2017  (cost=0.00..30.40 rows=2040 width=12)
+```
+
+See how it's querying our partitions in addition to the parent table? Now, watch what happens when we put a WHERE condition on the `year` column:
+
+```ruby
+Comment.where(year: 2016).explain
+=> EXPLAIN for: SELECT "comments".* FROM "comments" WHERE "comments"."year" = $1 [["year", 2016]]
+                              QUERY PLAN
+----------------------------------------------------------------------
+ Append  (cost=0.00..35.50 rows=11 width=12)
+   ->  Seq Scan on comments  (cost=0.00..0.00 rows=1 width=12)
+         Filter: (year = 2016)
+   ->  Seq Scan on comments_2016  (cost=0.00..35.50 rows=10 width=12)
+         Filter: (year = 2016)
+```
+
+Notice how it never looked at the `comments_2017` table? That's the magic of partitions.
+
+Now, there's one remaining issue. When we insert data into the `comments` table, we need it to route to be inserted into a partition instead of the actual table. For that, we can create a trigger:
+
+```ruby
+add_partition_trigger :comments, :comments_by_year, [
+  { if:    'NEW.year = 2016', insert: :comments_2016 },
+  { elsif: 'NEW.year = 2017', insert: :comments_2017 },
+  { else:  "RAISE EXECEPTION 'comments_by_year recieived an unexpected value: %', NEW.year;" }
+]
+```
+
+If the new record has a `year` of 2016, it'll be inserted into the `comments_2016` table. If the `year` is 2017, it'll be inserted into the `comments_2017` table. Otherwise, the trigger will throw an error.
+
+Now, imagine a year goes by and you need to add another partition for `2018`. You'll need to add the partition and update the trigger:
+
+```ruby
+add_partition :comments, :comments_2018, check: 'NEW.year = 2018'
+
+update_partition_trigger :comments, :comments_by_year, [
+  { if:    'NEW.year = 2016', insert: :comments_2016 },
+  { elsif: 'NEW.year = 2017', insert: :comments_2017 },
+  { elsif: 'NEW.year = 2018', insert: :comments_2018 },
+  { else:  "RAISE EXECEPTION 'comments_by_year recieived an unexpected value: %', NEW.year;" }
+]
+```
+
+## Caveats
+
+* You'll have to set `config.active_record.schema_format = :sql`. PgPartition doesn't support the use of `schema.rb`.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/rzane/pg_partitions.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,23 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+namespace :db do
+  task :setup do
+    system 'createdb pg_partitions_dev'
+    system 'createdb pg_partitions_test'
+  end
+
+  task :reset do
+    system 'dropdb pg_partitions_dev'
+    system 'dropdb pg_partitions_test'
+    Rake::Task['db:setup'].invoke
+  end
+end
+
+task default: :test

data/bin/console

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'pg_partitions'
+require 'active_record'
+require 'pry'
+
+ActiveRecord::Base.establish_connection(
+  adapter: 'postgresql',
+  database: 'pg_partitions_dev'
+)
+
+class Migration < ActiveRecord::Migration[5.1]
+  include PgPartitions
+
+  def change
+    create_table :comments do |t|
+      t.integer :year
+    end
+
+    add_partition :comments, :comments_2016, check: 'year = 2016'
+    add_partition :comments, :comments_2017, check: 'year = 2017'
+  end
+end
+
+ActiveRecord::Migration.run Migration
+
+class Comment < ActiveRecord::Base
+end
+
+binding.pry

data/bin/setup

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+rake db:setup

data/lib/pg_partitions.rb

@@ -0,0 +1,49 @@
+require 'active_support/core_ext/string'
+
+require 'pg_partitions/version'
+require 'pg_partitions/sql'
+
+module PgPartitions
+  def add_partition(table, name, check:)
+    statement = SQL::Partition.new(table, check)
+    create_table(name, id: false, options: statement.to_sql)
+  end
+
+  def add_partition_trigger(table, name, conditions)
+    insert_trigger    = SQL::Trigger.new(table, "#{name}_insert", 'BEFORE INSERT')
+    delete_function   = SQL::DeleteFunction.new(table, "#{name}_delete")
+    delete_trigger    = SQL::Trigger.new(table, "#{name}_delete", 'AFTER INSERT')
+
+    reversible do |dir|
+      dir.up do
+        update_partition_trigger(table, name, conditions)
+        execute insert_trigger.to_sql
+
+        execute delete_function.to_sql
+        execute delete_trigger.to_sql
+      end
+
+      dir.down do
+        drop_partition_trigger(table, name)
+      end
+    end
+  end
+
+  def update_partition_trigger(table, name, conditions)
+    insert_conditions = SQL::If.new(conditions)
+    insert_function   = SQL::InsertFunction.new(
+      table,
+      "#{name}_insert",
+      insert_conditions.to_sql
+    )
+
+    execute insert_function.to_sql
+  end
+
+  def drop_partition_trigger(table, name)
+    execute "DROP TRIGGER #{name}_insert ON #{table}"
+    execute "DROP FUNCTION #{name}_insert()"
+    execute "DROP TRIGGER #{name}_delete ON #{table}"
+    execute "DROP FUNCTION #{name}_delete()"
+  end
+end

data/lib/pg_partitions/sql.rb

@@ -0,0 +1,82 @@
+module PgPartitions
+  module SQL
+    class Partition < Struct.new(:table, :check)
+      def to_sql
+        "(LIKE #{table} INCLUDING ALL, CHECK (#{check})) INHERITS (#{table})"
+      end
+    end
+
+    class If < Struct.new(:conditions)
+      def to_sql
+        if conditions.empty?
+          raise ArgumentError, 'You must provide at least one condition'
+        end
+
+        lines = conditions.map do |opts|
+          if opts.key? :if
+            build_condition :if, opts
+          elsif opts.key? :elsif
+            build_condition :elsif, opts
+          else opts.key? :else
+            "ELSE\n  #{opts[:else]}"
+          end
+        end
+
+        lines << 'END IF;'
+        lines.join("\n") << "\n"
+      end
+
+      private
+
+      def build_condition(key, opts)
+        then_sql = opts.fetch :then do
+          "INSERT INTO #{opts.fetch(:insert)} VALUES(NEW.*) RETURNING * INTO result;"
+        end
+
+        "#{key.to_s.upcase} (#{opts[key]}) THEN\n  #{then_sql}"
+      end
+    end
+
+    class Function < Struct.new(:table, :name, :body)
+      def to_sql
+        <<~SQL
+          CREATE OR REPLACE FUNCTION #{name}()
+          RETURNS TRIGGER AS $$
+          DECLARE
+            result #{table}%rowtype;
+          BEGIN
+          #{body.indent(2)}
+            RETURN result;
+          END;
+          $$
+          LANGUAGE plpgsql;
+        SQL
+      end
+    end
+
+    class InsertFunction < Function
+    end
+
+    class DeleteFunction < Function
+      def initialize(table, name)
+        super(table, name, delete_statement(table))
+      end
+
+      private
+
+      def delete_statement(table)
+        "DELETE FROM ONLY #{table} WHERE id = NEW.id RETURNING * INTO result;"
+      end
+    end
+
+    class Trigger < Struct.new(:table, :name, :event)
+      def to_sql
+        <<~SQL
+          CREATE TRIGGER #{name}
+          #{event} ON #{table}
+          FOR EACH ROW EXECUTE PROCEDURE #{name}();
+        SQL
+      end
+    end
+  end
+end

data/lib/pg_partitions/version.rb

@@ -0,0 +1,3 @@
+module PgPartitions
+  VERSION = "0.1.0"
+end

data/pg_partitions.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pg_partitions/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pg_partitions"
+  spec.version       = PgPartitions::VERSION
+  spec.authors       = ["Ray Zane"]
+  spec.email         = ["ray@promptworks.com"]
+
+  spec.summary       = %q{ActiveRecord::Migration utility for creating partitions in PostgreSQL.}
+  spec.homepage      = "https://github.com/rzane/pg_partitions"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "pg"
+  spec.add_development_dependency "activerecord"
+end

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification
+name: pg_partitions
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ray Zane
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-09-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- ray@promptworks.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/pg_partitions.rb
+- lib/pg_partitions/sql.rb
+- lib/pg_partitions/version.rb
+- pg_partitions.gemspec
+homepage: https://github.com/rzane/pg_partitions
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.10
+signing_key: 
+specification_version: 4
+summary: ActiveRecord::Migration utility for creating partitions in PostgreSQL.
+test_files: []