pg_party 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee226c4523ca2b7099761ad517be99938078f7977281d88920beff47396f0c51
-  data.tar.gz: 2843ee1eb9fbe92e9998d8da0a3a20dfa989fa6c6dfcf3a261b80bac0caa47da
+  metadata.gz: f0edf666f9d4e7d3a7af82990be55d58b970949c5a549c1d8f1e1aeb2c066b09
+  data.tar.gz: 8c1164c72ffec0f00a892f9f4cf7028969a7aa39d056f4aae5e05674651a6142
 SHA512:
-  metadata.gz: 6cb4e5e42f08dcca3ec065c5d73656b6075e5a647d60722607cb4a9ff3cfeee24a94f4048544f3442284c816e7e37fe196579b9037e1ddacaaf1136e440efd4a
-  data.tar.gz: fdb593463dace1bdcfd844cac214cb6e6489dc7cd533226b90d11e506955ff48ad959fa80320423f3485637f89e3f6dbaeec5812ec0c1f640eb2d620e91de6dc
+  metadata.gz: 60baccd0471cd1628714d9d10e918d5b3be2708d07850bedf8751323cd91737321f93441fec5a576346de409580c3a7f2bb99a384c7edb0b1c70b0e96a2be84b
+  data.tar.gz: 12126fee3f2eae1b3a70771c79a07cab254b061c3b11ed20fbc8095f7521db05672219e93437f7a3cbb062b4ed95dd56aa2f09ae8e032b7a26cfaab7f6878def

data/README.md

@@ -61,6 +61,20 @@ These values can be accessed and set via `PgParty.config` and `PgParty.configure
 - `schema_exclude_partitions`
   - Whether to exclude child partitions in `rake db:structure:dump`
   - Default: `true`
+- `create_template_tables`
+  - Whether to create template tables by default. Use the `template:` option when creating partitioned tables to override this default.
+  - Default: `true`
+- `create_with_primary_key`
+  - Whether to add primary key constraints to partitioned (parent) tables by default.
+    * This is not supported for Postgres 10 but is recommended for Postgres 11
+    * Primary key constraints must include all partition keys, for example: `primary_key: [:id, :created_at], partition_key: :created_at`
+    * Partition keys cannot use expressions
+    * Can be overridden via the `create_with_primary_key:` option when creating partitioned tables
+  - Default: `false`
+- `include_subpartitions_in_partition_list`
+  - Whether to include nested subpartitions in the result of `YourModelClass.partiton_list` mby default.
+    You can always pass the `include_subpartitions:` option to override this.
+  - Default: `false` (for backward compatibility)
 
 Note that caching is done in-memory for each process of an application. Attaching / detaching partitions _will_ clear the cache, but only for the process that initiated the request. For multi-process web servers, it is recommended to use a TTL or disable caching entirely.
 
@@ -71,6 +85,10 @@ Note that caching is done in-memory for each process of an application. Attachin
 PgParty.configure do |c|
   c.caching_ttl = 60
   c.schema_exclude_partitions = false
+  c.include_subpartitions_in_partition_list = true
+  # Postgres 11+ users starting fresh may want to use below options
+  c.create_template_tables = false
+  c.create_with_primary_key = true
 end
 ```
 
@@ -88,25 +106,63 @@ These methods are available in migrations as well as `ActiveRecord::Base#connect
 - `create_list_partition`
   - Create partitioned table using the _list_ partitioning method
   - Required args: `table_name`, `partition_key:`
+- `create_hash_partition` (Postgres 11+)
+  - Create partitioned table using the _hash_ partitioning method
+  - Required args: `table_name`, `partition_key:`
 - `create_range_partition_of`
   - Create partition in _range_ partitioned table with partition key between _range_ of values
   - Required args: `table_name`, `start_range:`, `end_range:`
+  - Create a subpartition by specifying a `partition_type:` of `:range`, `:list`, or `:hash` and a `partition_key:`
 - `create_list_partition_of`
   - Create partition in _list_ partitioned table with partition key in _list_ of values
   - Required args: `table_name`, `values:`
+  - Create a subpartition by specifying a `partition_type:` of `:range`, `:list`, or `:hash` and a `partition_key:`
+- `create_hash_partition_of` (Postgres 11+)
+  - Create partition in _hash_ partitioned table for partition keys with hashed values having a specific remainder
+  - Required args: `table_name`, `modulus:`, `remainder`
+  - Create a subpartition by specifying a `partition_type:` of `:range`, `:list`, or `:hash` and a `partition_key:`
+  - Note that all partitions in a _hash_ partitioned table should have the same modulus. See [Examples](#examples) for more info.
+- `create_default_partition_of` (Postgres 11+)
+  - Create a default partition for values not falling in the range or list constraints of any other partitions
+  - Required args: `table_name`
 - `attach_range_partition`
   - Attach existing table to _range_ partitioned table with partition key between _range_ of values
   - Required args: `parent_table_name`, `child_table_name`, `start_range:`, `end_range:`
 - `attach_list_partition`
   - Attach existing table to _list_ partitioned table with partition key in _list_ of values
   - Required args: `parent_table_name`, `child_table_name`, `values:`
+- `attach_hash_partition` (Postgres 11+)
+  - Attach existing table to _hash_ partitioned table with partition key hashed values having a specific remainder
+  - Required args: `parent_table_name`, `child_table_name`, `modulus:`, `remainder`
+- `attach_default_partition` (Postgres 11+)
+  - Attach existing table as the _default_ partition
+  - Required args: `parent_table_name`, `child_table_name`
 - `detach_partition`
   - Detach partition from both _range and list_ partitioned tables
   - Required args: `parent_table_name`, `child_table_name`
 - `create_table_like`
   - Clone _any_ existing table
   - Required args: `table_name`, `new_table_name`
-
+- `partitions_for_table_name`
+  - List all attached partitions for a given table
+  - Required args: `table_name`, `include_subpartitions:` (true or false)
+- `parent_for_table_name`
+  - Fetch the parent table for a partition
+  - Required args: `table_name`
+  - Pass optional `traverse: true` to return the top-level table in the hierarchy (for subpartitions)
+  - Returns `nil` if the table is not a partition / has no parent
+- `table_partitioned?`
+  - Returns true if the table is partitioned (false for non-partitioned tables and partitions themselves)
+  - Required args: `table_name`
+- `add_index_on_all_partitions`
+  - Recursively add an index to all partitions and subpartitions of `table_name` using Postgres's ADD INDEX CONCURRENTLY
+    algorithm which adds the index in a non-blocking manner.
+  - Required args: `table_name`, `column_name` (all `add_index` arguments are supported)
+  - Use the `in_threads:` option to add indexes in parallel threads when there are many partitions. A value of 2 to 4
+    may be reasonable for tables with many large partitions and hosts with 4+ CPUs/cores.
+  - Use `disable_ddl_transaction!` in your migration to disable transactions when using this command with `in_threads:`
+    or `algorithm: :concurrently`.
+    
 #### Examples
 
 Create _range_ partitioned table on `created_at::date` with two partitions:
@@ -157,20 +213,130 @@ class CreateSomeListRecord < ActiveRecord::Migration[5.1]
      create_list_partition_of \
        :some_list_records,
        values: 101..200
+    
+    # default partition support is available in Postgres 11 or higher
+     create_default_partition_of \
+       :some_list_records
+  end
+end
+```
+
+Create _hash_ partitioned table on `id` with two partitions (Postgres 11+ required):
+  * A hash partition can be used to spread keys evenly(ish) across partitions
+  * `modulus:` should always equal the total number of partitions planned for the table
+  * `remainder:` is an integer which should be in the range of 0 to modulus-1
+
+```ruby
+class CreateSomeHashRecord < ActiveRecord::Migration[5.1]
+  def up
+    # symbol is used for partition keys referring to individual columns
+    # create_with_primary_key: true, template: false on Postgres 11 will rely on PostgreSQL's native partition schema
+    # management vs this gem's template tables
+    create_hash_partition :some_hash_records, partition_key: :id, create_with_primary_key: true, template: false do |t|
+      t.text :some_value
+      t.timestamps
+    end
+
+    # without name argument, child partition created as "some_list_records_<hash>"
+    create_hash_partition_of \
+      :some_hash_records,
+      modulus: 2,
+      remainder: 0
+
+    # without name argument, child partition created as "some_list_records_<hash>"
+    create_hash_partition_of \
+      :some_hash_records,
+      modulus: 2,
+      remainder: 1
   end
 end
 ```
 
+Advanced example with subpartitioning: Create _list_ partitioned table on `id` subpartitioned by _range_ on `created_at`
+with default partitions
+* We can use Postgres 11's support for primary keys vs template tables, using the composite primary key `[:id, :created_at]`
+  to ensure all partition keys are present in the primary key
+* Default partitions are only supported in Postgres 11+
+
+```ruby
+class CreateSomeListSubpartitionedRecord < ActiveRecord::Migration[5.1]
+  def up
+    # when specifying a composite primary key, the primary keys must be specified as columns
+    create_list_partition \
+      :some_list_subpartitioned_records,
+      partition_key: [:id],
+      primary_key: [:id, :created_at],
+      template: false,
+      create_with_primary_key: true do |t|
+
+      t.integer :id
+      t.text :some_value
+      t.timestamps
+    end
+
+    create_default_partition_of \
+      :some_list_subpartitioned_records,
+      name: :some_list_subpartitioned_records_default,
+      partition_type: :range,
+      partition_key: :created_at
+
+    create_range_partition_of \
+      :some_list_subpartitioned_records_default,
+      name: :some_list_subpartitioned_records_default_2019,
+      start_range: '2019-01-01',
+      end_range: '2019-12-31T23:59:59'
+    
+    create_default_partition_of \
+      :some_list_subpartitioned_records_default
+
+    create_list_partition_of \
+      :some_list_subpartitioned_records,
+      name: :some_list_subpartitioned_records_1,
+      values: 1..100,
+      partition_type: :range,
+      partition_key: :created_at
+  
+    create_range_partition_of \
+      :some_list_subpartitioned_records_1,
+      name: :some_list_subpartitioned_records_1_2019,
+      start_range: '2019-01-01',
+      end_range: '2019-12-31T23:59:59'
+
+    create_default_partition_of
+      :some_list_subpartitioned_records_1
+
+     create_list_partition_of \
+       :some_list_subpartitioned_records,
+       name: :some_list_subpartitioned_records_2,
+       values: 101..200,
+       partition_type: :range,
+       partition_key: :created_at
+
+    create_range_partition_of \
+      :some_list_subpartitioned_records_2,
+      name: :some_list_subpartitioned_records_2_2019,
+      start_range: '2019-01-01',
+      end_range: '2019-12-31T23:59:59'
+
+    create_default_partition_of \
+      :some_list_subpartitioned_records_2
+  end
+end
+```
+
+#### Template Tables
 Unfortunately, PostgreSQL 10 doesn't support indexes on partitioned tables.
 However, individual _partitions_ can have indexes.
 To avoid explicit index creation for _every_ new partition, we've introduced the idea of template tables.
 For every call to `create_list_partition` and `create_range_partition`, a clone `<table_name>_template` is created.
 Indexes, constraints, etc. created on the template table will propagate to new partitions in calls to `create_list_partition_of` and `create_range_partition_of`:
+* Subpartitions will correctly clone from template tables if a template table exists for the top-level ancestor
+* When using Postgres 11 or higher, you may wish to disable template tables and use the native features instead, see [Configuration](#configuration)
 
 ```ruby
 class CreateSomeListRecord < ActiveRecord::Migration[5.1]
   def up
-    # template table creation is enabled by default - use "template: false" to opt-out
+    # template table creation is enabled by default - use "template: false" or the config option to opt-out
     create_list_partition :some_list_records, partition_key: :id do |t|
       t.integer :some_foreign_id
       t.text :some_value
@@ -193,6 +359,8 @@ class CreateSomeListRecord < ActiveRecord::Migration[5.1]
 end
 ```
 
+#### Attaching Existing Tables as Partitions
+
 Attach an existing table to a _range_ partitioned table:
 
 ```ruby
@@ -220,6 +388,20 @@ class AttachListPartition < ActiveRecord::Migration[5.1]
 end
 ```
 
+Attach an existing table to a _hash_ partitioned table:
+
+```ruby
+class AttachHashPartition < ActiveRecord::Migration[5.1]
+  def up
+    attach_hash_partition \
+      :some_hash_records,
+      :some_existing_table,
+      modulus: 2,
+      remainder: 1
+  end
+end
+```
+
 Detach a partition from any partitioned table:
 
 ```ruby
@@ -230,6 +412,31 @@ class DetachPartition < ActiveRecord::Migration[5.1]
 end
 ```
 
+#### Safely cascading `add_index` commands
+Postgres 11+ will automatically cascade CREATE INDEX operations to partitions and subpartitions, however
+CREATE INDEX CONCURRENTLY is not supported, meaning table locks will be taken on each table while the new index is built.
+Postgres 10 provides no way to cascade index creation natively.
+* The `add_index_on_all_partitions` method solves for these limitations by recursively creating the specified
+  index on all partitions and subpartitions. Index names on individual partitions will include a hash suffix to avoid conflicts.
+* On Postgres 11+, the created indices are correctly attached to an index on the parent table
+* On Postgres 10, if you are using [Template Tables](#template-tables-for-postgres-10), you will want to add the index to the template table separately.
+* This command can also be used on subpartitions to cascade targeted indices starting at one level of the table hierarchy
+
+```ruby
+class AddSomeValueIndexToSomeListRecord < ActiveRecord::Migration[5.1]
+  # add_index_on_all_partitions with in_threads option may not be used within a transaction
+  # (also, algorithm: :concurrently cannot be used within a transaction)
+  disable_ddl_transaction!
+
+  def up
+    add_index :some_records_template, :some_value # Only if using Postgres 10 with template tables
+
+    # Pass the `in_threads:` option to create indices in parallel across multiple Postgres connections
+    add_index_on_all_partitions :some_records, :some_value, algorithm: :concurrently, in_threads: 4
+  end
+end
+```
+
 For more examples, take a look at the Combustion schema definition and integration specs:
 
 - https://github.com/rkrage/pg_party/blob/master/spec/dummy/db/schema.rb
@@ -250,12 +457,15 @@ Class methods available to _all_ ActiveRecord models:
 - `list_partition_by`
   - Configure a model backed by a _list_ partitioned table
   - Required arg: `key` (partition key column) or block returning partition key expression
+- `hash_partition_by`
+  - Configure a model backed by a _hash_ partitioned table
+  - Required arg: `key` (partition key column) or block returning partition key expression
 
 Class methods available to both _range and list_ partitioned models:
 
 - `partitions`
   - Retrieve a list of currently attached partitions
-  - No arguments
+  - Optional `include_subpartitions:` argument to include all subpartitions in the returned list
 - `in_partition`
   - Retrieve an ActiveRecord model scoped to an individual partition
   - Required arg: `child_table_name`
@@ -280,6 +490,16 @@ Class methods available to _list_ partitioned models:
 - `partition_key_in`
   - Query for records where partition key in _list_ of values
   - Required arg: list of `values`
+  
+
+Class methods available to _hash_ partitioned models:
+
+- `create_partition`
+  - Dynamically create new partition with hashed partition key divided by _modulus_ equals _remainder_
+  - Required arg: `modulus:`, `remainder:`
+- `partition_key_in`
+  - Query for records where partition key in _list_ of values (method operates the same as for _list_ partitions above)
+  - Required arg: list of `values`
 
 #### Examples
 
@@ -301,6 +521,15 @@ class SomeListRecord < ApplicationRecord
 end
 ```
 
+Configure model backed by a _hash_ partitioned table to get access to the methods described above:
+
+```ruby
+class SomeHashRecord < ApplicationRecord
+  # symbol is used for partition keys referring to individual columns
+  hash_partition_by :id
+end
+```
+
 Dynamically create new partition from _range_ partitioned model:
 
 ```ruby
@@ -315,19 +544,26 @@ Dynamically create new partition from _list_ partitioned model:
 SomeListRecord.create_partition(values: 200..300)
 ```
 
+Dynamically create new partition from _hash_ partitioned model:
+
+```ruby
+# additional options include: "name:" and "primary_key:"
+SomeHashRecord.create_partition(modulus: 2, remainder: 1)
+```
+
 For _range_ partitioned model, query for records where partition key in _range_ of values:
 
 ```ruby
 SomeRangeRecord.partition_key_in("2019-06-08", "2019-06-10")
 ```
 
-For _list_ partitioned model, query for records where partition key in _list_ of values:
+For _list_ and _hash_ partitioned models, query for records where partition key in _list_ of values:
 
 ```ruby
 SomeListRecord.partition_key_in(1, 2, 3, 4)
 ```
 
-For both _range and list_ partitioned models, query for records matching partition key:
+For all partitioned models, query for records matching partition key:
 
 ```ruby
 SomeRangeRecord.partition_key_eq(Date.current)
@@ -335,15 +571,15 @@ SomeRangeRecord.partition_key_eq(Date.current)
 SomeListRecord.partition_key_eq(100)
 ```
 
-For both _range and list_ partitioned models, retrieve currently attached partitions:
+For all partitioned models, retrieve currently attached partitions:
 
 ```ruby
 SomeRangeRecord.partitions
 
-SomeListRecord.partitions
+SomeListRecord.partitions(include_subpartitions: true) # Include nested subpartitions
 ```
 
-For both _range and list_ partitioned models, retrieve ActiveRecord model scoped to individual partition:
+For both all partitioned models, retrieve ActiveRecord model scoped to individual partition:
 
 ```ruby
 SomeRangeRecord.in_partition(:some_range_records_partition_name)

data/lib/pg_party/adapter/abstract_methods.rb

@@ -11,6 +11,10 @@ module PgParty
         raise "#create_list_partition is not implemented"
       end
 
+      def create_hash_partition(*)
+        raise "#create_hash_partition is not implemented"
+      end
+
       def create_range_partition_of(*)
         raise "#create_range_partition_of is not implemented"
       end
@@ -19,6 +23,14 @@ module PgParty
         raise "#create_list_partition_of is not implemented"
       end
 
+      def create_hash_partition_of(*)
+        raise "#create_hash_partition_of is not implemented"
+      end
+
+      def create_default_partition_of(*)
+        raise "#create_default_partition_of is not implemented"
+      end
+
       def create_table_like(*)
         raise "#create_table_like is not implemented"
       end
@@ -31,9 +43,33 @@ module PgParty
         raise "#attach_list_partition is not implemented"
       end
 
+      def attach_hash_partition(*)
+        raise "#attach_hash_partition is not implemented"
+      end
+
+      def attach_default_partition(*)
+        raise "#attach_default_partition is not implemented"
+      end
+
       def detach_partition(*)
         raise "#detach_partition is not implemented"
       end
+
+      def parent_for_table_name(*)
+        raise "#parent_for_table_name is not implemented"
+      end
+
+      def partitions_for_table_name(*)
+        raise "#partitions_for_table_name is not implemented"
+      end
+
+      def add_index_on_all_partitions(*)
+        raise "#add_index_on_all_partitions is not implemented"
+      end
+
+      def table_partitioned?(*)
+        raise "#table_partitioned? is not implemented"
+      end
     end
   end
 end

data/lib/pg_party/adapter/postgresql_methods.rb

@@ -14,6 +14,10 @@ module PgParty
         PgParty::AdapterDecorator.new(self).create_list_partition(*args, &blk)
       end
 
+      ruby2_keywords def create_hash_partition(*args, &blk)
+        PgParty::AdapterDecorator.new(self).create_hash_partition(*args, &blk)
+      end
+
       ruby2_keywords def create_range_partition_of(*args)
         PgParty::AdapterDecorator.new(self).create_range_partition_of(*args)
       end
@@ -22,6 +26,14 @@ module PgParty
         PgParty::AdapterDecorator.new(self).create_list_partition_of(*args)
       end
 
+      ruby2_keywords def create_hash_partition_of(*args)
+        PgParty::AdapterDecorator.new(self).create_hash_partition_of(*args)
+      end
+
+      ruby2_keywords def create_default_partition_of(*args)
+        PgParty::AdapterDecorator.new(self).create_default_partition_of(*args)
+      end
+
       ruby2_keywords def create_table_like(*args)
         PgParty::AdapterDecorator.new(self).create_table_like(*args)
       end
@@ -34,9 +46,33 @@ module PgParty
         PgParty::AdapterDecorator.new(self).attach_list_partition(*args)
       end
 
+      ruby2_keywords def attach_hash_partition(*args)
+        PgParty::AdapterDecorator.new(self).attach_hash_partition(*args)
+      end
+
+      ruby2_keywords def attach_default_partition(*args)
+        PgParty::AdapterDecorator.new(self).attach_default_partition(*args)
+      end
+
       ruby2_keywords def detach_partition(*args)
         PgParty::AdapterDecorator.new(self).detach_partition(*args)
       end
+
+      ruby2_keywords def partitions_for_table_name(*args)
+        PgParty::AdapterDecorator.new(self).partitions_for_table_name(*args)
+      end
+
+      ruby2_keywords def parent_for_table_name(*args)
+        PgParty::AdapterDecorator.new(self).parent_for_table_name(*args)
+      end
+
+      ruby2_keywords def add_index_on_all_partitions(*args)
+        PgParty::AdapterDecorator.new(self).add_index_on_all_partitions(*args)
+      end
+
+      ruby2_keywords def table_partitioned?(*args)
+        PgParty::AdapterDecorator.new(self).table_partitioned?(*args)
+      end
     end
   end
 end

data/lib/pg_party/adapter_decorator.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 
 require "digest"
+require 'parallel'
 
 module PgParty
   class AdapterDecorator < SimpleDelegator
+    SUPPORTED_PARTITION_TYPES = %i[range list hash].freeze
+
     def initialize(adapter)
       super(adapter)
 
@@ -18,6 +21,10 @@ module PgParty
       create_partition(table_name, :list, partition_key, **options, &blk)
     end
 
+    def create_hash_partition(table_name, partition_key:, **options, &blk)
+      create_partition(table_name, :hash, partition_key, **options, &blk)
+    end
+
     def create_range_partition_of(table_name, start_range:, end_range:, **options)
       create_partition_of(table_name, range_constraint_clause(start_range, end_range), **options)
     end
@@ -26,17 +33,42 @@ module PgParty
       create_partition_of(table_name, list_constraint_clause(values), **options)
     end
 
+    def create_hash_partition_of(table_name, modulus:, remainder:, **options)
+      create_partition_of(table_name, hash_constraint_clause(modulus, remainder), **options)
+    end
+
+    def create_default_partition_of(table_name, **options)
+      create_partition_of(table_name, nil, default_partition: true, **options)
+    end
+
     def create_table_like(table_name, new_table_name, **options)
-      primary_key = options.fetch(:primary_key) { calculate_primary_key(table_name) }
+      primary_key           = options.fetch(:primary_key) { calculate_primary_key(table_name) }
+      partition_key         = options.fetch(:partition_key, nil)
+      partition_type        = options.fetch(:partition_type, nil)
+      create_with_pks       = options.fetch(
+                                :create_with_primary_key,
+                                PgParty.config.create_with_primary_key
+                              )
+
+      validate_primary_key(primary_key) unless create_with_pks
+      if partition_type
+        validate_supported_partition_type!(partition_type)
+        raise ArgumentError, '`partition_key` is required when specifying a partition_type' unless partition_key
+      end
 
-      validate_primary_key(primary_key)
+      like_option = if !partition_type || create_with_pks
+                      'INCLUDING ALL'
+                    else
+                      'INCLUDING ALL EXCLUDING INDEXES'
+                    end
 
       execute(<<-SQL)
         CREATE TABLE #{quote_table_name(new_table_name)} (
-          LIKE #{quote_table_name(table_name)} INCLUDING ALL
-        )
+          LIKE #{quote_table_name(table_name)} #{like_option}
+        ) #{partition_type ? partition_by_clause(partition_type, partition_key) : nil}
       SQL
 
+      return if partition_type
       return if !primary_key
       return if has_primary_key?(new_table_name)
 
@@ -54,6 +86,20 @@ module PgParty
       attach_partition(parent_table_name, child_table_name, list_constraint_clause(values))
     end
 
+    def attach_hash_partition(parent_table_name, child_table_name, modulus:, remainder:)
+      attach_partition(parent_table_name, child_table_name, hash_constraint_clause(modulus, remainder))
+    end
+
+    def attach_default_partition(parent_table_name, child_table_name)
+      execute(<<-SQL)
+        ALTER TABLE #{quote_table_name(parent_table_name)}
+        ATTACH PARTITION #{quote_table_name(child_table_name)}
+        DEFAULT
+      SQL
+
+      PgParty.cache.clear!
+    end
+
     def detach_partition(parent_table_name, child_table_name)
       execute(<<-SQL)
         ALTER TABLE #{quote_table_name(parent_table_name)}
@@ -63,23 +109,101 @@ module PgParty
       PgParty.cache.clear!
     end
 
-    private
+    def partitions_for_table_name(table_name, include_subpartitions:, _accumulator: [])
+      select_values(%[
+          SELECT pg_inherits.inhrelid::regclass::text
+          FROM pg_tables
+          INNER JOIN pg_inherits
+            ON pg_tables.tablename::regclass = pg_inherits.inhparent::regclass
+          WHERE pg_tables.schemaname = current_schema() AND
+          pg_tables.tablename = #{quote(table_name)}
+                    ]).each_with_object(_accumulator) do |partition, acc|
+        acc << partition
+        next unless include_subpartitions
+
+        partitions_for_table_name(partition, include_subpartitions: true, _accumulator: acc)
+      end
+    end
 
-    def create_partition(table_name, type, partition_key, **options)
-      modified_options = options.except(:id, :primary_key, :template)
-      template         = options.fetch(:template, true)
-      id               = options.fetch(:id, :bigserial)
-      primary_key      = options.fetch(:primary_key) { calculate_primary_key(table_name) }
+    def parent_for_table_name(table_name, traverse: false)
+      parent = select_values(%[
+          SELECT pg_inherits.inhparent::regclass::text
+          FROM pg_tables
+          INNER JOIN pg_inherits
+            ON pg_tables.tablename::regclass = pg_inherits.inhrelid::regclass
+          WHERE pg_tables.schemaname = current_schema() AND
+          pg_tables.tablename = #{quote(table_name)}
+      ]).first
+      return parent if parent.nil? || !traverse
+
+      while (parents_parent = parent_for_table_name(parent)) do
+        parent = parents_parent
+      end
 
-      validate_primary_key(primary_key)
+      parent
+    end
+
+    def add_index_on_all_partitions(table_name, column_name, in_threads: nil, **options)
+      if in_threads && open_transactions > 0
+        raise ArgumentError, '`in_threads:` cannot be used within a transaction. If running in a migration, use '\
+              '`disable_ddl_transaction!` and break out this operation into its own migration.'
+      end
 
-      modified_options[:id]      = false
-      modified_options[:options] = "PARTITION BY #{type.to_s.upcase} (#{quote_partition_key(partition_key)})"
+      index_name, index_type, index_columns, index_options, algorithm, using = add_index_options(
+          table_name, column_name, options
+        )
+
+      # Postgres limits index name to 63 bytes (characters). We will use 8 characters for a `_random_suffix`
+      # on partitions to ensure no conflicts, leaving 55 chars for the specified index name
+      raise ArgumentError 'index name is too long - must be 55 characters or fewer' if index_name.length > 55
+
+      recursive_add_index(
+        table_name: table_name,
+        index_name: index_name,
+        index_type: index_type,
+        index_columns: index_columns,
+        index_options: index_options,
+        algorithm: algorithm,
+        using: using,
+        in_threads: in_threads
+      )
+    end
+
+    def table_partitioned?(table_name)
+      select_values(%[
+        SELECT relkind FROM pg_catalog.pg_class AS c
+        JOIN pg_catalog.pg_namespace AS ns ON c.relnamespace = ns.oid
+        WHERE relname = #{quote(table_name)} AND nspname = current_schema()
+      ]).first == 'p'
+    end
+
+    private
+
+    def create_partition(table_name, type, partition_key, **options)
+      modified_options      = options.except(:id, :primary_key, :template, :create_with_primary_key)
+      template              = options.fetch(:template, PgParty.config.create_template_tables)
+      id                    = options.fetch(:id, :bigserial)
+      primary_key           = options.fetch(:primary_key) { calculate_primary_key(table_name) }
+      create_with_pks       = options.fetch(
+                                :create_with_primary_key,
+                                PgParty.config.create_with_primary_key
+                              )
+
+      validate_supported_partition_type!(type)
+
+      if create_with_pks
+        modified_options[:primary_key] = primary_key
+        modified_options[:id] = id
+      else
+        validate_primary_key(primary_key)
+        modified_options[:id] = false
+      end
+      modified_options[:options] = partition_by_clause(type, partition_key)
 
       create_table(table_name, modified_options) do |td|
-        if id == :uuid
+        if !modified_options[:id] && id == :uuid
           td.column(primary_key, id, null: false, default: uuid_function)
-        elsif id
+        elsif !modified_options[:id] && id
           td.column(primary_key, id, null: false)
         end
 
@@ -87,11 +211,16 @@ module PgParty
       end
 
       # Rails 4 has a bug where uuid columns are always nullable
-      change_column_null(table_name, primary_key, false) if id == :uuid
+      change_column_null(table_name, primary_key, false) if !modified_options[:id] && id == :uuid
 
       return unless template
 
-      create_table_like(table_name, template_table_name(table_name), primary_key: id && primary_key)
+      create_table_like(
+        table_name,
+        template_table_name(table_name),
+        primary_key: id && primary_key,
+        create_with_primary_key: create_with_pks
+      )
     end
 
     def create_partition_of(table_name, constraint_clause, **options)
@@ -99,13 +228,21 @@ module PgParty
       primary_key         = options.fetch(:primary_key) { calculate_primary_key(table_name) }
       template_table_name = template_table_name(table_name)
 
+      validate_default_partition_support! if options[:default_partition]
+
       if schema_cache.data_source_exists?(template_table_name)
-        create_table_like(template_table_name, child_table_name, primary_key: false)
+        create_table_like(template_table_name, child_table_name, primary_key: false,
+                          partition_type: options[:partition_type], partition_key: options[:partition_key])
       else
-        create_table_like(table_name, child_table_name, primary_key: primary_key)
+        create_table_like(table_name, child_table_name, primary_key: primary_key,
+                          partition_type: options[:partition_type], partition_key: options[:partition_key])
       end
 
-      attach_partition(table_name, child_table_name, constraint_clause)
+      if options[:default_partition]
+        attach_default_partition(table_name, child_table_name)
+      else
+        attach_partition(table_name, child_table_name, constraint_clause)
+      end
 
       child_table_name
     end
@@ -120,6 +257,82 @@ module PgParty
       PgParty.cache.clear!
     end
 
+    def recursive_add_index(table_name:, index_name:, index_type:, index_columns:, index_options:, using:, algorithm:,
+                            in_threads: nil, _parent_index_name: nil, _created_index_names: [])
+      partitions = partitions_for_table_name(table_name, include_subpartitions: false)
+      updated_name = _created_index_names.empty? ? index_name : generate_index_name(index_name, table_name)
+
+      # If this is a partitioned table, add index ONLY on this table.
+      if table_partitioned?(table_name)
+        add_index_only(table_name, type: index_type, name: updated_name, using: using, columns: index_columns,
+                       options: index_options)
+        _created_index_names << updated_name
+
+        parallel_map(partitions, in_threads: in_threads) do |partition_name|
+          recursive_add_index(
+            table_name: partition_name,
+            index_name: index_name,
+            index_type: index_type,
+            index_columns: index_columns,
+            index_options: index_options,
+            using: using,
+            algorithm: algorithm,
+            _parent_index_name: updated_name,
+            _created_index_names: _created_index_names
+          )
+        end
+      else
+        _created_index_names << updated_name # Track as created before execution of concurrent index command
+        add_index_from_options(table_name, name: updated_name, type: index_type, algorithm: algorithm, using: using,
+                               columns: index_columns, options: index_options)
+      end
+
+      attach_child_index(updated_name, _parent_index_name) if _parent_index_name
+
+      return true if index_valid?(updated_name)
+
+      raise 'index creation failed - an index was marked invalid'
+    rescue => e
+      # Clean up any indexes created so this command can be retried later
+      drop_indices_if_exist(_created_index_names)
+      raise e
+    end
+
+    def attach_child_index(child, parent)
+      return unless postgres_major_version >= 11
+
+      execute "ALTER INDEX #{quote_column_name(parent)} ATTACH PARTITION #{quote_column_name(child)}"
+    end
+
+    def add_index_only(table_name, type:, name:, using:, columns:, options:)
+      return unless postgres_major_version >= 11
+
+      execute "CREATE #{type} INDEX #{quote_column_name(name)} ON ONLY "\
+              "#{quote_table_name(table_name)} #{using} (#{columns})#{options}"
+    end
+
+    def add_index_from_options(table_name, name:, type:, algorithm:, using:, columns:, options:)
+      execute "CREATE #{type} INDEX #{algorithm} #{quote_column_name(name)} ON "\
+              "#{quote_table_name(table_name)} #{using} (#{columns})#{options}"
+    end
+
+    def drop_indices_if_exist(index_names)
+      index_names.uniq.each { |name| execute "DROP INDEX IF EXISTS #{quote_column_name(name)}" }
+    end
+
+    def parallel_map(arr, in_threads:)
+      return [] if arr.empty?
+      return arr.map { |item| yield(item) } unless in_threads && in_threads > 1
+
+      if ActiveRecord::Base.connection_pool.size <= in_threads
+        raise ArgumentError, 'in_threads: must be lower than your database connection pool size'
+      end
+
+      Parallel.map(arr, in_threads: in_threads) do |item|
+        ActiveRecord::Base.connection_pool.with_connection { yield(item) }
+      end
+    end
+
     # Rails 5.2 now returns boolean literals
     # This causes partition creation to fail when the constraint clause includes a boolean
     # Might be a PostgreSQL bug, but for now let's revert to the old quoting behavior
@@ -157,27 +370,69 @@ module PgParty
     end
 
     def template_table_name(table_name)
-      "#{table_name}_template"
+      "#{parent_for_table_name(table_name, traverse: true) || table_name}_template"
     end
 
     def range_constraint_clause(start_range, end_range)
       "FROM (#{quote_collection(start_range)}) TO (#{quote_collection(end_range)})"
     end
 
+    def hash_constraint_clause(modulus, remainder)
+      "WITH (MODULUS #{modulus.to_i}, REMAINDER #{remainder.to_i})"
+    end
+
     def list_constraint_clause(values)
       "IN (#{quote_collection(values.try(:to_a) || values)})"
     end
 
+    def partition_by_clause(type, partition_key)
+      "PARTITION BY #{type.to_s.upcase} (#{quote_partition_key(partition_key)})"
+    end
+
     def uuid_function
       try(:supports_pgcrypto_uuid?) ? "gen_random_uuid()" : "uuid_generate_v4()"
     end
 
     def hashed_table_name(table_name, key)
-      "#{table_name}_#{Digest::MD5.hexdigest(key)[0..6]}"
+      return "#{table_name}_#{Digest::MD5.hexdigest(key)[0..6]}" if key
+
+      # use _default suffix for default partitions (without a constraint clause)
+      "#{table_name}_default"
+    end
+
+    def index_valid?(index_name)
+      select_values(
+        "SELECT relname FROM pg_class, pg_index WHERE pg_index.indisvalid = false AND "\
+          "pg_index.indexrelid = pg_class.oid AND relname = #{quote(index_name)}"
+      ).empty?
+    end
+
+    def generate_index_name(index_name, table_name)
+      "#{index_name}_#{Digest::MD5.hexdigest(table_name)[0..6]}"
+    end
+
+    def validate_supported_partition_type!(partition_type)
+      if (sym = partition_type.to_s.downcase.to_sym) && sym.in?(SUPPORTED_PARTITION_TYPES)
+        return if sym != :hash || postgres_major_version >= 11
+
+        raise NotImplementedError, 'Hash partitions are only available in Postgres 11 or higher'
+      end
+
+      raise ArgumentError, "Supported partition types are #{SUPPORTED_PARTITION_TYPES.join(', ')}"
+    end
+
+    def validate_default_partition_support!
+      return if postgres_major_version >= 11
+
+      raise NotImplementedError, 'Default partitions are only available in Postgres 11 or higher'
     end
 
     def supports_partitions?
-      __getobj__.send(:postgresql_version) >= 100000
+      postgres_major_version >= 10
+    end
+
+    def postgres_major_version
+      __getobj__.send(:postgresql_version)/10000
     end
   end
 end

data/lib/pg_party/cache.rb

@@ -9,7 +9,7 @@ module PgParty
     def initialize
       # automatically initialize a new hash when
       # accessing an object id that doesn't exist
-      @store = Hash.new { |h, k| h[k] = { models: {}, partitions: nil } }
+      @store = Hash.new { |h, k| h[k] = { models: {}, partitions: nil, partitions_with_subpartitions: nil } }
     end
 
     def clear!
@@ -24,10 +24,11 @@ module PgParty
       LOCK.synchronize { fetch_value(@store[key][:models], child_table.to_sym, block) }
     end
 
-    def fetch_partitions(key, &block)
+    def fetch_partitions(key, include_subpartitions, &block)
       return block.call unless caching_enabled?
+      sub_key = include_subpartitions ? :partitions_with_subpartitions : :partitions
 
-      LOCK.synchronize { fetch_value(@store[key], :partitions, block) }
+      LOCK.synchronize { fetch_value(@store[key], sub_key, block) }
     end
 
     private

data/lib/pg_party/config.rb

@@ -5,12 +5,18 @@ module PgParty
     attr_accessor \
       :caching,
       :caching_ttl,
-      :schema_exclude_partitions
+      :schema_exclude_partitions,
+      :create_template_tables,
+      :create_with_primary_key,
+      :include_subpartitions_in_partition_list
 
     def initialize
       @caching = true
       @caching_ttl = -1
       @schema_exclude_partitions = true
+      @create_template_tables = true
+      @create_with_primary_key = false
+      @include_subpartitions_in_partition_list = false
     end
   end
 end

data/lib/pg_party/model/hash_methods.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "pg_party/model_decorator"
+require "ruby2_keywords"
+
+module PgParty
+  module Model
+    module HashMethods
+      ruby2_keywords def create_partition(*args)
+        PgParty::ModelDecorator.new(self).create_hash_partition(*args)
+      end
+
+      ruby2_keywords def partition_key_in(*args)
+        PgParty::ModelDecorator.new(self).hash_partition_key_in(*args)
+      end
+    end
+  end
+end

data/lib/pg_party/model/list_methods.rb

@@ -10,6 +10,10 @@ module PgParty
         PgParty::ModelDecorator.new(self).create_list_partition(*args)
       end
 
+      ruby2_keywords def create_default_partition(*args)
+        PgParty::ModelDecorator.new(self).create_default_partition(*args)
+      end
+
       ruby2_keywords def partition_key_in(*args)
         PgParty::ModelDecorator.new(self).list_partition_key_in(*args)
       end

data/lib/pg_party/model/methods.rb

@@ -13,6 +13,10 @@ module PgParty
         PgParty::ModelInjector.new(self, *key, &blk).inject_list_methods
       end
 
+      def hash_partition_by(*key, &blk)
+        PgParty::ModelInjector.new(self, *key, &blk).inject_hash_methods
+      end
+
       def partitioned?
         try(:partition_key).present?
       end

data/lib/pg_party/model/range_methods.rb

@@ -10,6 +10,10 @@ module PgParty
         PgParty::ModelDecorator.new(self).create_range_partition(*args)
       end
 
+      ruby2_keywords def create_default_partition(*args)
+        PgParty::ModelDecorator.new(self).create_default_partition(*args)
+      end
+
       ruby2_keywords def partition_key_in(*args)
         PgParty::ModelDecorator.new(self).range_partition_key_in(*args)
       end

data/lib/pg_party/model/shared_methods.rb

@@ -6,13 +6,15 @@ module PgParty
   module Model
     module SharedMethods
       def reset_primary_key
-        if self != base_class
-          base_class.primary_key
-        elsif partition_name = partitions.first
-          in_partition(partition_name).get_primary_key(base_class.name)
-        else
-          get_primary_key(base_class.name)
-        end
+        return base_class.primary_key if self != base_class
+
+        partitions = partitions(include_subpartitions: true)
+        return get_primary_key(base_class.name) if partitions.empty?
+
+        first_partition = partitions.detect { |p| !connection.table_partitioned?(p) }
+        raise 'No leaf partitions exist for this model. Create a partition to contain your data' unless first_partition
+
+        in_partition(first_partition).get_primary_key(base_class.name)
       end
 
       def table_exists?
@@ -21,8 +23,8 @@ module PgParty
         connection.schema_cache.data_source_exists?(target_table)
       end
 
-      def partitions
-        PgParty::ModelDecorator.new(self).partitions
+      def partitions(*args)
+        PgParty::ModelDecorator.new(self).partitions(*args)
       end
 
       def in_partition(*args)

data/lib/pg_party/model_decorator.rb

@@ -62,15 +62,11 @@ module PgParty
       end
     end
 
-    def partitions
-      PgParty.cache.fetch_partitions(cache_key) do
-        connection.select_values(<<-SQL)
-          SELECT pg_inherits.inhrelid::regclass::text
-          FROM pg_tables
-          INNER JOIN pg_inherits
-            ON pg_tables.tablename::regclass = pg_inherits.inhparent::regclass
-          WHERE pg_tables.tablename = #{connection.quote(table_name)}
-        SQL
+    alias_method :hash_partition_key_in, :list_partition_key_in
+
+    def partitions(include_subpartitions: PgParty.config.include_subpartitions_in_partition_list)
+      PgParty.cache.fetch_partitions(cache_key, include_subpartitions) do
+        connection.partitions_for_table_name(table_name, include_subpartitions: include_subpartitions)
       end
     rescue
       []
@@ -95,6 +91,23 @@ module PgParty
       create_partition(:create_list_partition_of, table_name, **modified_options)
     end
 
+    def create_hash_partition(modulus:, remainder:, **options)
+      modified_options = options.merge(
+        modulus: modulus,
+        remainder: remainder,
+        primary_key: primary_key,
+      )
+
+      create_partition(:create_hash_partition_of, table_name, **modified_options)
+    end
+
+    def create_default_partition(**options)
+      modified_options = options.merge(
+        primary_key: primary_key,
+      )
+      create_partition(:create_default_partition_of, table_name, **modified_options)
+    end
+
     private
 
     def create_partition(migration_method, table_name, **options)

data/lib/pg_party/model_injector.rb

@@ -20,6 +20,12 @@ module PgParty
       inject_methods_for(PgParty::Model::ListMethods)
     end
 
+    def inject_hash_methods
+      require "pg_party/model/hash_methods"
+
+      inject_methods_for(PgParty::Model::HashMethods)
+    end
+
     private
 
     def inject_methods_for(mod)

data/lib/pg_party/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgParty
-  VERSION = "1.3.0"
+  VERSION = "1.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_party
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Ryan Krage
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-12 00:00:00.000000000 Z
+date: 2020-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -44,6 +44,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.2
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -235,6 +249,7 @@ files:
 - lib/pg_party/cache.rb
 - lib/pg_party/config.rb
 - lib/pg_party/hacks/postgresql_database_tasks.rb
+- lib/pg_party/model/hash_methods.rb
 - lib/pg_party/model/list_methods.rb
 - lib/pg_party/model/methods.rb
 - lib/pg_party/model/range_methods.rb