partitioned 0.8.0 → 1.0.1

data/lib/partitioned/bulk_methods_mixin.rb

@@ -1,77 +1,56 @@
 module Partitioned
+  #
+  # MixIn used to extend ActiveRecord::Base classes implementing bulk insert and update operations
+  # through {#create_many} and {#update_many}.  {Partitioned::PartitionedBase} classes are extended
+  # with these methods by default.
+  #
+  # @example to use in non-{Partitioned::PartitionedBase} class:
+  #   class Company < ActiveRecord::Base
+  #     extend Partitioned::BulkMethodsMixin
+  #   end
+  #
   module BulkMethodsMixin
+    # exception thrown when row data structures are inconsistent between rows in single call to {#create_many} or {#update_many}
     class BulkUploadDataInconsistent < StandardError
       def initialize(model, table_name, expected_columns, found_columns, while_doing)
         super("#{model.name}: for table: #{table_name}; #{expected_columns} != #{found_columns}; #{while_doing}")
       end
     end
-    #
+
     # BULK creation of many rows
     #
-    # rows: an array of hashtables of data to insert into the database
-    #       each hashtable must have the same number of keys (and same
-    #       names for each key).
-    #
-    # options:
-    #   :slice_size = 1000
-    #   :returning = nil
-    #   :check_consistency = true
-    #
-    # examples:
-    #  first example didn't uses more options.
-    #
-    # rows = [{
-    #   :name => 'Keith',
-    #   :salary => 1000,
-    # },
-    # {
-    #   :name => 'Alex',
-    #   :salary => 2000,
-    # }]
-    #
-    # Employee.create_many(rows)
-    #
-    #  this second example uses :returning option
-    #  to returns key values
-    #
-    # rows = [{
-    #   :name => 'Keith',
-    #   :salary => 1000,
-    # },
-    # {
-    #   :name => 'Alex',
-    #   :salary => 2000,
-    # }]
-    #
-    # options = {
-    #   :returning => [:id]
-    # }
-    #
-    # Employee.create_many(rows, options) returns [#<Employee id: 1>, #<Employee id: 2>]
-    #
-    #  third example uses :slice_size option.
-    #  Slice_size - is an integer that specifies how many
-    #  records will be created in a single SQL query.
-    #
-    # rows = [{
-    #   :name => 'Keith',
-    #   :salary => 1000,
-    # },
-    # {
-    #   :name => 'Alex',
-    #   :salary => 2000,
-    # },
-    # {
-    #   :name => 'Mark',
-    #   :salary => 3000,
-    # }]
-    #
-    # options = {
-    #   :slice_size => 2
-    # }
-    #
-    # Employee.create_many(rows, options) will generate two insert queries
-    #
+    # @example no options used
+    #   rows = [
+    #         { :name => 'Keith', :salary => 1000 },
+    #         { :name => 'Alex', :salary => 2000 }
+    #   ]
+    #   Employee.create_many(rows)
+    #
+    # @example with :returning option to returns key value
+    #   rows = [
+    #         { :name => 'Keith', :salary => 1000 },
+    #         { :name => 'Alex', :salary => 2000 }
+    #   ]
+    #   options = { :returning => [:id] }
+    #   Employee.create_many(rows, options)
+    #   [#<Employee id: 1>, #<Employee id: 2>]
+    #
+    # @example with :slice_size option (will generate two insert queries)
+    #   rows = [
+    #         { :name => 'Keith', :salary => 1000 },
+    #         { :name => 'Alex', :salary => 2000 },
+    #         { :name => 'Mark', :salary => 3000 }
+    #   ]
+    #   options = { :slice_size => 2 }
+    #   Employee.create_many(rows, options)
+    #
+    # @param [Array<Hash>] rows ([]) data to be inserted into database
+    # @param [Hash] options ({}) options for bulk inserts
+    # @option options [Integer] :slice_size (1000) how many records will be created in a single SQL query
+    # @option options [Boolean] :check_consitency (true) ensure some modicum of sanity on the incoming dataset, specifically: does each row define the same set of key/value pairs
+    # @option options [Array or String] :returning (nil) list of fields to return.
+    # @return [Array<Hash>] rows returned from DB as option[:returning] requests
+    # @raise [BulkUploadDataInconsistent] raised when key/value pairs between rows are inconsistent (check disabled with option :check_consistency)
     def create_many(rows, options = {})
       return [] if rows.blank?
       options[:slice_size] = 1000 unless options.has_key?(:slice_size)
@@ -127,90 +106,56 @@ module Partitioned
     #
     # BULK updates of many rows
     #
-    # rows: an array of hashtables of data to insert into the database
-    #       each hashtable must have the same number of keys (and same
-    #       names for each key).
-    #
-    # options:
-    #   :slice_size = 1000
-    #   :returning = nil
-    #   :set_array = from first row passed in
-    #   :check_consistency = true
-    #   :where = '"#{table_name}.id = datatable.id"'
-    #
-    # examples:
-    #  this first example uses "set_array" to add the value of "salary"
-    #  to the specific employee's salary
-    #  the default where clause is to match IDs so, it works here.
-    # rows = [{
-    #   :id => 1,
-    #   :salary => 1000,
-    # },
-    # {
-    #   :id => 10,
-    #   :salary => 2000,
-    # },
-    # {
-    #   :id => 23,
-    #   :salary => 2500,
-    # }]
-    #
-    # options = {
-    #   :set_array => '"salary = datatable.salary"'
-    # }
-    #
-    # Employee.update_many(rows, options)
-    #
-    #
-    #  this versions sets the where clause to match Salaries.
-    # rows = [{
-    #   :id => 1,
-    #   :salary => 1000,
-    #   :company_id => 10
-    # },
-    # {
-    #   :id => 10,
-    #   :salary => 2000,
-    #   :company_id => 12
-    # },
-    # {
-    #   :id => 23,
-    #   :salary => 2500,
-    #   :company_id => 5
-    # }]
-    #
-    # options = {
-    #   :set_array => '"company_id = datatable.company_id"',
-    #   :where => '"#{table_name}.salary = datatable.salary"'
-    # }
-    #
-    # Employee.update_many(rows, options)
-    #
-    #
-    #  this version sets the where clause to the KEY of the hash passed in
-    # and the set_array is generated from the VALUES
-    #
-    # rows = {
-    #   { :id => 1 } => {
-    #     :salary => 100000,
-    #     :company_id => 10
-    #   },
-    #   { :id => 10 } => {
-    #     :salary => 110000,
-    #     :company_id => 12
-    #   },
-    #   { :id => 23 } => {
-    #     :salary => 90000,
-    #     :company_id => 5
+    # @return [Array<Hash>] rows returned from DB as option[:returning] requests
+    # @raise [BulkUploadDataInconsistent] raised when key/value pairs between rows are inconsistent (check disabled with option :check_consistency)
+    # @param [Hash] options ({}) options for bulk inserts
+    # @option options [Integer] :slice_size (1000) how many records will be created in a single SQL query
+    # @option options [Boolean] :check_consitency (true) ensure some modicum of sanity on the incoming dataset, specifically: does each row define the same set of key/value pairs
+    # @option options [Array] :returning (nil) list of fields to return.
+    # @option options [String] :returning (nil) single field to return.
+    #
+    # @overload update_many(rows = [], options = {})
+    #   @param [Array<Hash>] rows ([]) data to be updated
+    #   @option options [String] :set_array (built from first row passed in) the set clause
+    #   @option options [String] :where ('"#{table_name}.id = datatable.id"') the where clause
+    #
+    # @overload update_many(rows = {}, options = {})
+    #   @param [Hash<Hash, Hash>] rows ({}) data to be updated
+    #   @option options [String] :set_array (built from the values in the first key/value pair of `rows`) the set clause
+    #   @option options [String] :where (built from the keys in the first key/value pair of `rows`) the where clause
+    #
+    # @example using "set_array" to add the value of "salary" to the specific employee's salary the default where clause matches IDs so, it works here.
+    #   rows = [
+    #     { :id => 1, :salary => 1000 },
+    #     { :id => 10, :salary => 2000 },
+    #     { :id => 23, :salary => 2500 }
+    #   ]
+    #   options = { :set_array => '"salary = datatable.salary"' }
+    #   Employee.update_many(rows, options)
+    #
+    # @example using where clause to match salary.
+    #   rows = [
+    #     { :id => 1, :salary => 1000, :company_id => 10 },
+    #     { :id => 10, :salary => 2000, :company_id => 12 },
+    #     { :id => 23, :salary => 2500, :company_id => 5 }
+    #   ]
+    #   options = {
+    #     :set_array => '"company_id = datatable.company_id"',
+    #     :where => '"#{table_name}.salary = datatable.salary"'
     #   }
-    # }
+    #   Employee.update_many(rows, options)
     #
-    # Employee.update_many(rows)
+    # @example setting where clause to the KEY of the hash passed in and the set_array is generated from the VALUES
+    #   rows = {
+    #     { :id => 1 } => { :salary => 100000, :company_id => 10 },
+    #     { :id => 10 } => { :salary => 110000, :company_id => 12 },
+    #     { :id => 23 } => { :salary => 90000, :company_id => 5 }
+    #   }
+    #   Employee.update_many(rows)
     #
-    # Remember that you should probably set updated_at using "updated = datatable.updated_at"
-    # or "updated_at = now()" in the set_array if you want to follow
-    # the standard active record model for time columns (and you have an updated_at column)
-
+    # @note Remember that you should probably set updated_at using "updated = datatable.updated_at"
+    #   or "updated_at = now()" in the set_array if you want to follow
+    #   the standard active record model for time columns (and you have an updated_at column)
     def update_many(rows, options = {})
       return [] if rows.blank?
       if rows.is_a?(Hash)

data/lib/partitioned/by_created_at.rb

@@ -1,11 +1,13 @@
 module Partitioned
   #
-  # partition tables by created_at grouping them by week, with
+  # Partition tables by created_at grouping them by week, with
   # a week defined as seven days starting on Monday.
   #
   class ByCreatedAt < ByWeeklyTimeField
     self.abstract_class = true
 
+    # the field to partition on, `created_at`
+    # @return [Symbol] the partition field: `created_at`
     def self.partition_time_field
       return :created_at
     end

data/lib/partitioned/by_foreign_key.rb

@@ -1,11 +1,16 @@
 module Partitioned
+  # Partitioned abstract class for all partitioned models based as a single integer field value that is used as a foreign key
   class ByForeignKey < ByIntegerField
     self.abstract_class = true
 
+    # the field to partition on
+    # @return [Integer] re-routed to {#self.partition_foreign_key}
     def self.partition_integer_field
       return partition_foreign_key
     end
 
+    # the field to partition on
+    # @return [String] the name of the foreign key field
     def self.partition_foreign_key
       raise MethodNotImplemented.new(self, :partition_foreign_key)
     end

data/lib/partitioned/by_id.rb

@@ -1,7 +1,7 @@
 module Partitioned
   #
-  # table partitioning by id.  this partitioning breaks up data by
-  # the value of its primary key.  a specific record's child table
+  # Table partitioning by id. this partitioning breaks up data by
+  # the value of its primary key. A specific record's child table
   # is determined by the number resulting from the integer math:
   #   ID / ById::partition_table_size * ById::partition_table_size
   # 
@@ -9,21 +9,27 @@ module Partitioned
     self.abstract_class = true
 
     #
-    # specific to this partitioning, we need to prefetch the primary key (id)
+    # Specific to this partitioning, we need to prefetch the primary key (id)
     # before we attempt to do the insert because the insert wants to know the
     # name of the specific child table to access.
     #
+    # @return [Boolean] true
     def self.prefetch_primary_key?
       return true
     end
 
     #
-    # the number of records in each child table.
+    # The number of records in each child table.
     #
+    # @return [Integer] the number of rows in a partition
     def self.partition_table_size
       return 10000000
     end
 
+    #
+    # The name of the field to partition on
+    #
+    # @return [String] the name of the field to partition on
     def self.partition_integer_field
       return :id
     end

data/lib/partitioned/by_integer_field.rb

@@ -1,15 +1,24 @@
 module Partitioned
+  #
+  # Partitioned abstract class for all partitioned models based as a single integer field value.
+  #
   class ByIntegerField < PartitionedBase
     self.abstract_class = true
 
+    # the size of each table
+    # @return [Integer] how many different values are in each partition
     def self.partition_table_size
       return 1
     end
 
+    # the name of the partition key field
+    # @return [String] the name of the field
     def self.partition_integer_field
       raise MethodNotImplemented.new(self, :partition_integer_field)
     end
 
+    # the normalized key value for a given key value
+    # @return [Integer] the normalized value
     def self.partition_normalize_key_value(integer_field_value)
       return integer_field_value / partition_table_size * partition_table_size
     end

data/lib/partitioned/by_monthly_time_field.rb

@@ -1,15 +1,22 @@
 module Partitioned
   #
-  # partition tables by a time field grouping them by week, with
+  # Partition tables by a time field grouping them by week, with
   # a week defined as seven days starting on Monday.
   #
   class ByMonthlyTimeField < ByTimeField
     self.abstract_class = true
 
+    # Normalize a partition key value by month.
+    #
+    # @param [Time] time_value the time value to normalize
+    # @return [Time] the value normalized
     def self.partition_normalize_key_value(time_value)
       return time_value.at_beginning_of_month
     end
 
+    # The size of the partition table, a month
+    # 
+    # @return [Integer] the size of this partition
     def self.partition_table_size
       return 1.month
     end

data/lib/partitioned/by_time_field.rb

@@ -1,16 +1,20 @@
 module Partitioned
   #
-  # partition tables by a time field grouping them by day
+  # Partition tables by a time field grouping them by day.
   #
   class ByTimeField < PartitionedBase
     self.abstract_class = true
 
     #
-    # generate an enumerable that represents all the dates between
-    # start_date and end_date skipping step
+    # Generate an enumerable that represents all the dates between
+    # start_date and end_date skipping step.
     #
-    # this can be used to calls that take an enumerable like create_infrastructure
+    # This can be used to calls that take an enumerable like create_infrastructure.
     #
+    # @param [Date] start_date the first date to generate the range from
+    # @param [Date] end_date the last date to generate the range from
+    # @param [Object] step (:default) number of values to advance (:default means use {#self.partition_table_size}).
+    # @return [Enumerable] the range generated
     def self.partition_generate_range(start_date, end_date, step = :default)
       step = partition_table_size if step == :default
       current_date = partition_normalize_key_value(start_date)
@@ -23,23 +27,27 @@ module Partitioned
     end
 
     #
-    # normalize the value to the current day
+    # Normalize the value to the current day.
     #
+    # @param [Time] time_value the partitioned key value
+    # @return [Time] time_value normalized
     def self.partition_normalize_key_value(time_value)
       return time_value.to_date
     end
 
     #
-    # the size of the partition, 1.day
+    # The size of the partition, 1.day
     #
+    # @return [Integer] the size of the partition
     def self.partition_table_size
       return 1.day
     end
 
     #
-    # abstract -- implement in a derived clas.
-    # the name of the time-related field we will use to partition child tables
+    # Abstract -- implement in a derived class.
+    # The name of the time-related field we will use to partition child tables.
     #
+    # @raise MethodNotImplemented
     def self.partition_time_field
       raise MethodNotImplemented.new(self, :partition_time_field)
     end

data/lib/partitioned/by_weekly_time_field.rb

@@ -1,15 +1,17 @@
 module Partitioned
   #
-  # partition tables by a time field grouping them by week, with
+  # Partition tables by a time field grouping them by week, with
   # a week defined as seven days starting on Monday.
   #
   class ByWeeklyTimeField < ByTimeField
     self.abstract_class = true
 
     #
-    # normalize a partition key value by week.  We've picked
-    # the begining of the week to key on, which is Monday.
+    # Normalize a partition key value by week. We've picked
+    # the beginning of the week to key on, which is Monday.
     #
+    # @param [Time] time_value the time value to normalize
+    # @return [Time] the value normalized
     def self.partition_normalize_key_value(time_value)
       return time_value.at_beginning_of_week
     end
@@ -17,6 +19,7 @@ module Partitioned
     #
     # The size of the partition table, 7 days (1.week)
     # 
+    # @return [Integer] the size of this partition
     def self.partition_table_size
       return 1.week
     end

data/lib/partitioned/multi_level/configurator/data.rb

@@ -1,6 +1,7 @@
 module Partitioned
   class MultiLevel
     module Configurator
+      # partitioning configuration information
       class Data < Partitioned::PartitionedBase::Configurator::Data
         attr_accessor :using_classes
 

data/lib/partitioned/multi_level/configurator/dsl.rb

@@ -1,7 +1,15 @@
 module Partitioned
   class MultiLevel
+    #
+    # Configuration manager for multi-level partitioning
+    # it supports, the front-end UI (a Domain Specific Language) using {Dsl}
+    # state using {Data}
+    # and a parser using {Reader}
+    #
     module Configurator
+      # The Domain Specific Language UI manager for multi level partitioning classes 
       class Dsl < Partitioned::PartitionedBase::Configurator::Dsl
+        # used to prevent use of invalid directives
         class InvalidForMultiLevelPartitioning < StandardError
           def initialize(model, dsl_key, remedy)
             super("#{model.name}: '#{dsl_key}' is not valid for multi-level partitioning.  #{remedy}")
@@ -18,10 +26,13 @@ module Partitioned
         #
         # Definition of classes which will be used at multi level partitioning.
         #
+        # @param [*Array<Class>] classes the classes, in order, used to partition this model
+        # @return [optional]
         def using_classes(*classes)
           data.using_classes += [*classes]
         end
 
+        # @raise [InvalidForMultiLevelPartitioning] always raised. `on` is not a valid DSL directive for multi-level partitioning
         def on(*ignored)
           raise InvalidForMultiLevelPartitioning.new(model, :on, "the partitioned keyword 'using' is used to define multi-level partitioned tables.")
         end

data/lib/partitioned/multi_level/configurator/reader.rb

@@ -1,8 +1,13 @@
 module Partitioned
   class MultiLevel
     module Configurator
+      # coalesces and parses all {Data} objects allowing the
+      # {PartitionManager} to request partitioning information froma
+      # centralized source from multi level partitioned models
       class Reader < Partitioned::PartitionedBase::Configurator::Reader
+        # configurator for a specific class level
         UsingConfigurator = Struct.new(:model, :sliced_class, :dsl)
+
         def initialize(most_derived_activerecord_class)
           super
           @using_classes = nil
@@ -12,6 +17,7 @@ module Partitioned
         #
         # The field used to partition child tables.
         #
+        # @return [Array<Symbol>] fields used to partition this model
         def on_fields
           unless @on_fields
             @on_fields = using_collect(&:on_field).map(&:to_sym)
@@ -72,10 +78,22 @@ module Partitioned
           return parts.join('_')
         end
 
+        # retrieve a specific configurator from an ordered list.  for multi-level partitioning
+        # we need to find the specific configurator for the partitioning level we are interested
+        # in managing.
+        #
+        # @param [Integer] index the partitioning level to query
+        # @return [Configurator] the configurator for the specific level queried
         def using_configurator(index)
           return using_class(index).configurator
         end
 
+        # retrieve a specific partitioning class from an ordered list.  for multi-level partitioning
+        # we need to find the specific {Partitioned::PartitionedBase} class for the partitioning level we are interested
+        # in managing.
+        #
+        # @param [Integer] index the partitioning level to query
+        # @return [{Partitioned::PartitionedBase}] the class for the specific level queried
         def using_class(index)
           return using_classes[index]
         end

data/lib/partitioned/multi_level/partition_manager.rb

@@ -1,10 +1,15 @@
 module Partitioned
   class MultiLevel
+    #
+    # the manger of partitioned requests for models partitioned multiple times
+    #
     class PartitionManager < Partitioned::PartitionedBase::PartitionManager
       #
-      # the once called function to prepare a parent table for partitioning as well
+      # The once called function to prepare a parent table for partitioning as well
       # as create the schema that the child tables will be placed in.
       #
+      # @param [Enumerable] enumerable (Array<Array>) the key values that should be used to create the parent partition tables.
+      # @return [optional]
       def create_infrastructure(enumerable = [[]])
         super()
         enumerable.each do |*partition_key_values|
@@ -15,11 +20,13 @@ module Partitioned
       protected
 
       #
-      # create a specific child table that does not currently
+      # Create a specific child table that does not currently
       # exist and whose schema (the schema that the table exists in)
       # also already exists (#create_infrastructure is designed to
       # create this).
       #
+      # @param [*Array<Object>] partition_key_values all key values needed to create a partition
+      # @return [optional]
       def create_new_partition(*partition_key_values)
         create_partition_table(*partition_key_values)
         if is_leaf_partition?(*partition_key_values)
@@ -31,14 +38,16 @@ module Partitioned
       end
 
       #
-      # is the table a child table without itself having any children.
+      # Is the table a child table without itself having any children.
       # generally leaf tables are where all indexes and foreign key
       # constraints will be placed because that is where the data will be.
       #
       # Non leaf tables will typically have a rule placed on them
-      # (via add_parent_table_rules) that prevents any inserts from occuring
+      # (via add_parent_table_rules) that prevents any inserts from occurring
       # on them.
       #
+      # @param [*Array<Object>] partition_key_values all key values specifying a given child table
+      # @return [Boolean] true if this partition should contain records
       def is_leaf_partition?(*partition_key_values)
         return partition_key_values.length == parent_table_class.configurator.on_fields.length
       end

data/lib/partitioned/multi_level.rb

@@ -1,6 +1,6 @@
 module Partitioned
   #
-  # table partitioning by a referenced id column which itself is partitioned
+  # Table partitioning by a referenced id column which itself is partitioned
   # further weekly by a date column.
   # 
   class MultiLevel < PartitionedBase
@@ -9,6 +9,8 @@ module Partitioned
     #
     # Normalize the values for the each of using class.
     #
+    # @param [Array<Object>] value the partition key values
+    # @return [Array<Object>] the normalized values for the key values passed in
     def self.partition_normalize_key_value(values)
       normalized_values = []
       [*values].each_with_index do |value,index|

data/lib/partitioned/partitioned_base/configurator/data.rb

@@ -2,9 +2,10 @@ module Partitioned
   class PartitionedBase
     module Configurator
       #
-      # the state configured by the Dsl and read by Reader
+      # The state configured by the Dsl and read by Reader.
       #
       class Data
+        # represents a SQL index
         class Index
           attr_accessor :field, :options
           def initialize(field, options = {})
@@ -12,6 +13,7 @@ module Partitioned
             @options = options
           end
         end
+        # represents a SQL foreign key reference
         class ForeignKey
           attr_accessor :referencing_field, :referenced_table, :referenced_field
           def initialize(referencing_field, referenced_table = nil, referenced_field = :id)
@@ -24,6 +26,13 @@ module Partitioned
             @referenced_field = referenced_field
           end
 
+          #
+          # Produce a table name from the name of the foreign key.  in rails, this really
+          # means "foo_id" should be mapped to "foos", and "company_id" should be mapped to
+          # "companies"
+          #
+          # @param [String] foreign_key_field the name of the foreign key field
+          # @return [String] the name of the table associated with the foreign key
           def self.foreign_key_to_foreign_table_name(foreign_key_field)
             return ActiveSupport::Inflector::pluralize(foreign_key_field.to_s.sub(/_id$/,''))
           end