partitioned 0.8.0

data/lib/partitioned/partitioned_base/configurator/reader.rb

@@ -0,0 +1,209 @@
+module Partitioned
+  class PartitionedBase
+    module Configurator
+      class Reader
+        attr_reader :model
+
+        def initialize(most_derived_activerecord_class)
+          @model = most_derived_activerecord_class
+          @configurators = nil
+          @on_fields = nil
+          @indexes = nil
+          @foreign_keys = nil
+          @check_constraint = nil
+
+          @schema_name = nil
+          @name_prefix = nil
+          @base_name = nil
+          @part_name = nil
+
+          @table_name = nil
+
+          @parent_table_schema_name = nil
+          @parent_table_name = nil
+
+          @encoded_name = nil
+        end
+
+        #
+        # The name of the schema that will contain all child tables.
+        #
+        def schema_name
+          unless @schema_name
+            @schema_name = collect_first(&:schema_name)
+          end
+          return @schema_name
+        end
+
+        #
+        # The field used to partition child tables.
+        #
+        def on_fields
+          unless @on_fields
+            @on_fields = collect(&:on_field).map(&:to_sym)
+          end
+          return @on_fields
+        end
+
+        #
+        # Define an index to be created on all (leaf-) child tables.
+        #
+        def indexes(*partition_key_values)
+          return collect_from_collection(*partition_key_values, &:indexes).inject({}) do |bag, data_index|
+            bag[data_index.field] = (data_index.options || {}) unless data_index.blank?
+            bag
+          end
+        end
+
+        #
+        # Define a foreign key on a (leaf-) child table.
+        #
+        def foreign_keys(*partition_key_values)
+          return collect_from_collection(*partition_key_values, &:foreign_keys).inject(Set.new) do |set,new_items|
+            if new_items.is_a? Array
+              set += new_items
+            else
+              set += [new_items]
+            end
+            set
+          end
+        end
+
+        #
+        # Define the check constraint for a given child table.
+        #
+        def check_constraint(*partition_key_values)
+          return collect_first(*partition_key_values, &:check_constraint)
+        end
+
+        #
+        # The table name of the table who is the direct ancestor of a child table.
+        #
+        def parent_table_name(*partition_key_values)
+          return collect_first(*partition_key_values, &:parent_table_name)
+        end
+
+        #
+        # The schema name of the table who is the direct ancestor of a child table.
+        #
+        def parent_table_schema_name(*partition_key_values)
+          return collect_first(*partition_key_values, &:parent_table_schema_name)
+        end
+
+        #
+        # The full name of a child table defined by the partition key values.
+        #
+        def table_name(*partition_key_values)
+          return collect_first(*partition_key_values, &:table_name)
+        end
+
+        #
+        # The name of the child table without the schema name or name prefix.
+        #
+        def base_name(*partition_key_values)
+          return collect_first(*partition_key_values, &:base_name)
+        end
+
+        #
+        # The prefix for the child table's name.
+        #
+        def name_prefix
+          unless @name_prefix
+            @name_prefix = collect_first(&:name_prefix)
+          end
+          return @name_prefix
+        end
+
+        #
+        # The child tables name without the schema name.
+        #
+        def part_name(*partition_key_values)
+          return collect_first(*partition_key_values, &:part_name)
+        end
+
+        #
+        # Define the order by clause used to list all child table names in order
+        # of "last to be used" to "oldest to have been used".
+        #
+        def last_partitions_order_by_clause
+          unless @last_partitions_order_by_clause
+            @last_partitions_order_by_clause = collect_first(&:last_partitions_order_by_clause)
+          end
+          return @last_partitions_order_by_clause
+        end
+
+
+        protected
+
+        def configurators
+          unless @configurators
+            @configurators = []
+            model.ancestors.each do |ancestor|
+              if ancestor.respond_to?(:configurator_dsl)
+                if ancestor::configurator_dsl
+                  @configurators << ancestor::configurator_dsl
+                end
+              end
+              break if ancestor == Partitioned::PartitionedBase
+            end
+          end
+          return @configurators
+        end
+
+        def collect(*partition_key_values, &block)
+          values = []
+          configurators.each do |configurator|
+            data = configurator.data
+
+            intermediate_value = block.call(data) rescue nil
+            if intermediate_value.is_a? Proc
+              values << intermediate_value.call(model, *partition_key_values)
+            elsif intermediate_value.is_a? String
+              field_value = partition_key_values.first
+              values << eval("\"#{intermediate_value}\"")
+            else
+              values << intermediate_value unless intermediate_value.blank?
+            end
+          end
+          return values
+        end
+
+        def collect_first(*partition_key_values, &block)
+          configurators.each do |configurator|
+            data = configurator.data
+            intermediate_value = block.call(data) rescue nil
+            if intermediate_value.is_a? Proc
+              return intermediate_value.call(model, *partition_key_values)
+            elsif intermediate_value.is_a? String
+              field_value = partition_key_values.first
+              return eval("\"#{intermediate_value}\"")
+            else
+              return intermediate_value unless intermediate_value.nil?
+            end
+          end
+          return nil
+        end
+
+        def collect_from_collection(*partition_key_values, &block)
+          values = []
+          configurators.each do |configurator|
+            data = configurator.data
+            intermediate_values = []
+            intermediate_values = block.call(data) rescue nil
+            [*intermediate_values].each do |intermediate_value|
+              if intermediate_value.is_a? Proc
+                values << intermediate_value.call(model, *partition_key_values)
+              elsif intermediate_value.is_a? String
+                field_value = partition_key_values.first
+                values << eval("\"#{intermediate_value}\"")
+              else
+                values << intermediate_value unless intermediate_value.blank?
+              end
+            end
+          end
+          return values
+        end
+      end
+    end
+  end
+end

data/lib/partitioned/partitioned_base/partition_manager.rb

@@ -0,0 +1,138 @@
+module Partitioned
+  class PartitionedBase
+    #
+    # PartitionManager
+    # interface for all requests made to build partition tables.
+    # these are typically delegated to us from the ActiveRecord class
+    # (partitioned_base.rb defines the forwarding)
+    class PartitionManager
+      attr_reader :parent_table_class
+
+      def initialize(parent_table_class)
+        @parent_table_class = parent_table_class
+      end
+
+      #
+      # drop partitions that are no longer necessary.
+      # uses #old_partition_key_values_set as the list of
+      # partitions to remove.
+      #
+      def drop_old_partitions
+        old_partition_key_values_set.each do |*partition_key_values|
+          drop_old_partition(*partition_key_values)
+        end
+      end
+
+      #
+      # create partitions that are needed (probably to handle data that
+      # will be inserted into the database within the next few weeks).
+      # uses #new_partition_key_value_set to determine the key values
+      # for the specific child tables to create.
+      #
+      def create_new_partitions
+        new_partition_key_values_set.each do |*partition_key_values|
+         create_new_partition(*partition_key_values)
+        end
+      end
+
+      #
+      # create any partition tables from a list.  the partition tables must
+      # not already exist and its schema must already exist.
+      #
+      def create_new_partition_tables(enumerable)
+        enumerable.each do |partition_key_values|
+          create_new_partition(*partition_key_values)
+        end
+      end
+
+      #
+      # the once called function to prepare a parent table for partitioning as well
+      # as create the schema that the child tables will be placed in.
+      #
+      def create_infrastructure
+        create_partition_schema
+        add_parent_table_rules
+      end
+
+      protected
+      #
+      # an array of key values (each key value is an array of keys) that represent
+      # the child partitions that should be created.
+      #
+      # used by #create_new_partitions and generally called once a day to update
+      # the database with new soon-to-be needed child tables
+      #
+      # typically overridden by the concrete class as this is pure business logic
+      #
+      def new_partition_key_values_set
+        []
+      end
+
+      #
+      # an array of key values (each key value is an array of keys) that represent
+      # the child partitions that should be dropped because they are no longer needed.
+      #
+      # used by #drop_old_partitions and generally called once a day to clean up
+      # unneeded child tables
+      #
+      # typically overridden by the concrete class as this is pure business logic
+      #
+      def old_partition_key_values_set
+        []
+      end
+
+      #
+      # remove a specific partition from the database given
+      # the key value(s) of its check constraint columns
+      #
+      def drop_old_partition(*partition_key_values)
+        drop_partition_table(*partition_key_values)
+      end
+
+      #
+      # create a specific child table that does not currently
+      # exist and whose schema (the schema that the table exists in)
+      # also already exists (#create_infrastructure is designed to
+      # create this).
+      #
+      def create_new_partition(*partition_key_values)
+        create_partition_table(*partition_key_values)
+        add_partition_table_index(*partition_key_values)
+        add_references_to_partition_table(*partition_key_values)
+      end
+
+      ##
+      # :method: drop_partition_table
+      # delegated to Partitioned::PartitionedBase::PartitionManager::SqlAdapter#drop_partition_table
+
+      ##
+      # :method: create_partition_table
+      # delegated to Partitioned::PartitionedBase::PartitionManager::SqlAdapter#create_partition_table
+
+      ##
+      # :method: add_partition_table_index
+      # delegated to Partitioned::PartitionedBase::PartitionManager::SqlAdapter#add_partition_table_index
+
+      ##
+      # :method: add_references_to_partition_table
+      # delegated to Partitioned::PartitionedBase::PartitionManager::SqlAdapter#add_references_to_partition_table
+
+      ##
+      # :method: create_partition_schema
+      # delegated to Partitioned::PartitionedBase::PartitionManager::SqlAdapter#create_partition_schema
+
+      ##
+      # :method: add_parent_table_rules
+      # delegated to Partitioned::PartitionedBase::PartitionManager::SqlAdapter#add_parent_table_rules
+
+      ##
+      # :method: partition_table_name
+      # delegated to Partitioned::PartitionedBase::PartitionManager::SqlAdapter#partition_table_name
+
+      extend Forwardable
+      def_delegators :parent_table_class, :sql_adapter
+      def_delegators :sql_adapter, :drop_partition_table, :create_partition_table, :add_partition_table_index,
+         :add_references_to_partition_table, :create_partition_schema, :add_parent_table_rules, :partition_table_name
+    end
+  end
+end

data/lib/partitioned/partitioned_base/sql_adapter.rb

@@ -0,0 +1,286 @@
+module Partitioned
+  class PartitionedBase
+    #
+    # SqlAdapter
+    # manages requests of partitioned tables.
+    #
+    class SqlAdapter
+      attr_reader :parent_table_class
+
+      def initialize(parent_table_class)
+        @parent_table_class = parent_table_class
+      end
+
+      #
+      # ensure our function for warning about improper partition usage is in place
+      #
+      # Name: always_fail_on_insert(text); Type: FUNCTION; Schema: public
+      #
+      # Used to raise an exception explaining why a specific insert (into a parent
+      # table which should never have records) should never be attempted.
+      #
+      def ensure_always_fail_on_insert_exists
+        sql = <<-SQL
+          CREATE OR REPLACE FUNCTION always_fail_on_insert(table_name text) RETURNS boolean
+              LANGUAGE plpgsql
+              AS $$
+           BEGIN
+             RAISE EXCEPTION 'partitioned table "%" does not support direct inserts, you should be inserting directly into child tables', table_name;
+             RETURN false;
+           END;
+          $$;
+        SQL
+        execute(sql)
+      end
+
+      #
+      # Child tables whose parent table is 'foos', typically exist in a schema named foos_partitions.
+      #
+      # *partition_key_values are needed here to support the use of multiple schemas to keep tables in.
+      #
+      def create_partition_schema(*partition_key_values)
+        create_schema(configurator.schema_name, :unless_exists => true)
+      end
+
+      #
+      # does a specific child partition exist
+      #
+      def partition_exists?(*partition_key_values)
+        return find(:first,
+                    :from => "pg_tables",
+                    :select => "count(*) as count",
+                    :conditions  => ["schemaname = ? and tablename = ?",
+                                      configurator.schema_name,
+                                      configurator.part_name(*partition_key_values)
+                    ]).count.to_i == 1
+      end
+
+      #
+      # returns an array of partition table names from last to first limited to
+      # the number of entries requested by its first parameter.
+      #
+      # the magic here is in the overridden method "last_n_partitions_order_by_clause"
+      # which is designed to order a list of partition table names (table names without
+      # their schema name) from last to first.
+      #
+      # if the child table names are the format "pYYYYMMDD" where YYYY is a four digit year, MM is
+      # a month number and DD is a day number, you would use the following to order from last to
+      # first:
+      #   tablename desc
+      #
+      # for child table names of the format "pXXXX" where XXXX is a number, you may want something like:
+      #   substring(tablename, 2)::integer desc
+      #
+      # for clarity, the sql executed is:
+      #    select tablename from pg_tables where schemaname = $1 order by $2 limit $3
+      # where:
+      #  $1 = the name of schema (foos_partitions)
+      #  $2 = the order by clause that would make the greatest table name listed first
+      #  $3 = the parameter 'how_many'
+      #
+      def last_n_partition_names(how_many = 1)
+        return find(:all,
+                    :from => "pg_tables",
+                    :select => :tablename,
+                    :conditions  => ["schemaname = ?", configurator.schema_name],
+                    :order => last_n_partitions_order_by_clause,
+                    :limit => how_many).map(&:tablename)
+      end
+
+      #
+      # override this or order the tables from last (greatest value? greatest date?) to first
+      #
+      def last_n_partitions_order_by_clause
+        return configurator.last_n_partitions_order_by_clause
+      end
+
+      #
+      # used to create the parent table rule to ensure
+      #
+      # this will cause an error on attempt to insert into the parent table
+      #
+      # we want all records to exist in one of the child tables so the
+      # query planner can optimize access to the records.
+      #
+      def add_parent_table_rules(*partition_key_values)
+        ensure_always_fail_on_insert_exists
+
+        insert_redirector_name = parent_table_rule_name("insert", "redirector", *partition_key_values)
+        sql = <<-SQL
+          CREATE OR REPLACE RULE #{insert_redirector_name} AS
+            ON INSERT TO #{configurator.parent_table_name(*partition_key_values)}
+            DO INSTEAD
+            (
+               SELECT always_fail_on_insert('#{configurator.parent_table_name(*partition_key_values)}')
+            )
+        SQL
+        execute(sql)
+      end
+
+      #
+      # the name of the table (schemaname.childtablename) given the check constraint values.
+      #
+      def partition_table_name(*partition_key_values)
+        return configurator.table_name(*partition_key_values)
+      end
+
+      #
+      # create a single child table.
+      #
+      def create_partition_table(*partition_key_values)
+        create_table(configurator.table_name(*partition_key_values), {
+                       :id => false,
+                       :options => "INHERITS (#{configurator.parent_table_name(*partition_key_values)})"
+                     }) do |t|
+          t.check_constraint configurator.check_constraint(*partition_key_values)
+        end
+      end
+
+      #
+      # remove a specific single child table
+      #
+      def drop_partition_table(*partition_key_values)
+        drop_table(configurator.table_name(*partition_key_values))
+      end
+
+      #
+      # add indexes that must exist on child tables.  Only leaf child tables
+      # need indexes as parent table indexes are not used in postgres.
+      #
+      def add_partition_table_index(*partition_key_values)
+        configurator.indexes(*partition_key_values).each do |field,options|
+          used_options = options.clone
+          unless used_options.has_key?(:name)
+            name = [*field].join('_')
+            used_options[:name] = used_options[:unique] ? unique_index_name(name, *partition_key_values) : index_name(name, *partition_key_values)
+          end
+          add_index(partition_table_name(*partition_key_values), field, used_options)
+        end
+      end
+
+      #
+      # used when creating the name of a SQL rule
+      #
+      def parent_table_rule_name(name, suffix = "rule", *partition_key_values)
+        return "#{configurator.parent_table_name(*partition_key_values).gsub(/[.]/, '_')}_#{name}_#{suffix}"
+      end
+
+      #
+      # used to create index names
+      #
+      def index_name(name, *partition_key_values)
+        return "#{configurator.part_name(*partition_key_values)}_#{name}_idx"
+      end
+
+      #
+      # used to create index names
+      #
+      def unique_index_name(name, *partition_key_values)
+        return "#{configurator.part_name(*partition_key_values)}_#{name}_udx"
+      end
+
+      #
+      # this is here for derived classes to set up references to added columns
+      # (or columns in the parent that need foreign key constraints).
+      #
+      # foreign keys are not inherited in postgres.  So, a parent table
+      # of the form:
+      #
+      #   -- this is the referenced table
+      #   create table companies
+      #   (
+      #       id               serial not null primary key,
+      #       created_at       timestamp not null default now(),
+      #       updated_at       timestamp,
+      #       name             text not null
+      #   );
+      #
+      #   -- this is the parent table
+      #   create table employees
+      #   (
+      #       id               serial not null primary key,
+      #       created_at       timestamp not null default now(),
+      #       updated_at       timestamp,
+      #       name             text not null,
+      #       company_id       integer not null references companies,
+      #       supervisor_id    integer not null references employees
+      #   );
+      #
+      #   -- some children
+      #   create table employees_of_company_1 ( CHECK ( company_id = 1 ) ) INHERITS (employees);
+      #   create table employees_of_company_2 ( CHECK ( company_id = 2 ) ) INHERITS (employees);
+      #   create table employees_of_company_3 ( CHECK ( company_id = 3 ) ) INHERITS (employees);
+      #
+      # since postgres does not inherit referential integrity from parent tables, the following
+      # insert will work:
+      #    insert into employees_of_company_1 (name, company_id, supervisor_id) values ('joe', 1, 10);
+      # even if there is no record in companies with id = 1 and there is no record in employees with id = 10
+      #
+      # for proper referential integrity handling you must do the following:
+      #   ALTER TABLE employees_of_company_1 add foreign key (company_id) references companies(id)
+      #   ALTER TABLE employees_of_company_2 add foreign key (company_id) references companies(id)
+      #   ALTER TABLE employees_of_company_3 add foreign key (company_id) references companies(id)
+      #
+      #   ALTER TABLE employees_of_company_1 add foreign key (supervisor_id) references employees_of_company_1(id)
+      #   ALTER TABLE employees_of_company_2 add foreign key (supervisor_id) references employees_of_company_2(id)
+      #   ALTER TABLE employees_of_company_3 add foreign key (supervisor_id) references employees_of_company_3(id)
+      #
+      # the second set of alter tables brings up a good another consideration about postgres references and partitions.
+      # postgres will not follow references to a child table.  So, a foreign key reference to "employees" in this
+      # set of alter statements would not work because postgres would expect the table "employees" to have
+      # the specific referenced record, but the record really exists in a child of employees.  So, the alter statement
+      # forces the reference check on the specific child table we know must contain this employees supervisor (since
+      # such a supervisor would have to work for the same company in our model).
+      #
+      def add_references_to_partition_table(*partition_key_values)
+        configurator.foreign_keys(*partition_key_values).each do |foreign_key|
+          add_foreign_key(partition_table_name(*partition_key_values),
+                          foreign_key.referencing_field,
+                          foreign_key.referenced_table,
+                          foreign_key.referenced_field)
+        end
+      end
+
+      ##
+      # :method: connection
+      # delegated to the connection of the parent table class
+
+      ##
+      # :method: execute
+      # delegated to the connection of the parent table class
+
+      ##
+      # :method: create_schema
+      # delegated to the connection of the parent table class
+
+      ##
+      # :method: drop_schema
+      # delegated to the connection of the parent table class
+
+      ##
+      # :method: add_index
+      # delegated to the connection of the parent table class
+
+      ##
+      # :method: remove_index
+      # delegated to the connection of the parent table class
+
+      ##
+      # :method: transaction
+      # delegated to the connection of the parent table class
+
+      ##
+      # :method: find_by_sql
+      # delegated to the connection of the parent table class
+
+      ##
+      # :method: find
+      # delegated to the connection of the parent table class
+
+      extend Forwardable
+      def_delegators :parent_table_class, :connection, :find_by_sql, :transaction, :find, :configurator
+      def_delegators :connection, :execute, :add_index, :remove_index, :create_schema, :drop_schema, :add_foreign_key,
+                     :create_table, :drop_table
+    end
+  end
+end