active_partition 0.1.0

data/README.md

@@ -0,0 +1,85 @@
+# ActivePartition
+
+The active_partition gem is a Ruby library designed for Rails application that provides functionality for partitioning data in a database table. Partitioning is a technique used to divide large datasets into smaller, more manageable chunks called partitions. This can improve query performance and make it easier to manage and maintain the data.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'active_partition'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install active_partition
+
+## Usage
+
+TODO: List all use-cases
+
+Apply partitioning to model.
+
+```ruby
+class Event < ActiveRecord::Base
+  include ActivePartition::Partitionable
+  # the name of partitioned colunn
+  self.partitioned_by = "created_at"
+  # You can change this range over time. from months to hours.
+  self.partition_range = 1.day
+
+  # You can choose 1 of the following 2 options
+  # Keep all partitions within a time period
+  self.retention_period = 1.month
+  # Keep last n partitions
+  self.retention_partition_count = 3
+end
+
+# auto create a new partition if needed.
+Event.create(created_at: Time.current)
+# create partition events_p_240404_04_1712203200_1712289600 from 2024-04-04 04:00:00 UTC to 2024-04-05 04:00:00 UTC
+
+# Delete expired partition (you can set cron job to run this command)
+Event.delete_expired_partitions
+
+# `premake` is also supported. create 3 1-month partitions
+Event.premake 1.month, 3
+# create partition outgoing_events_p_240801_04_1722484800_1725163200 from 2024-08-01 04:00:00 UTC to 2024-09-01 04:00:00 UTC
+# create partition outgoing_events_p_240901_04_1725163200_1727755200 from 2024-09-01 04:00:00 UTC to 2024-10-01 04:00:00 UTC
+# create partition outgoing_events_p_241001_04_1727755200_1730433600 from 2024-10-01 04:00:00 UTC to 2024-11-01 04:00:00 UTC
+
+# You can change premake period if needed. For example, create 2 1-year partition.
+Event.premake 1.year, 2
+# create partition outgoing_events_p_241101_04_1730433600_1761969600 from 2024-11-01 04:00:00 UTC to 2025-11-01 04:00:00 UTC
+# create partition outgoing_events_p_251101_04_1761969600_1793505600 from 2025-11-01 04:00:00 UTC to 2026-11-01 04:00:00 UTC
+```
+
+The partition name following the format
+```ruby
+"#{@table_name}_p_#{readable_from}_#{unix_from}_#{unix_to}"
+```
+
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/active_partition. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/active_partition/blob/main/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the ActivePartition project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/active_partition/blob/main/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task default: :test

data/active_partition.gemspec

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative "lib/active_partition/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "active_partition"
+  spec.version       = ActivePartition::VERSION
+  spec.authors       = ["Thien Tran"]
+  spec.email         = ["webmaster3t@gmail.com"]
+
+  spec.summary       = "An extension to ActiveRecord to support partitioned tables."
+  spec.description   = "Applying partition with flexible and risk-free by auto generate partitioned tables, manage partitions directly from ActiveRecord models."
+  spec.homepage      = "https://github.com/thien0291/active_partition"
+  spec.license       = "MIT"
+  spec.required_ruby_version = ">= 2.4.0"
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/thien0291/active_partition"
+  spec.metadata["changelog_uri"] = "https://github.com/thien0291/active_partition"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Uncomment to register a new dependency of your gem
+  spec.add_development_dependency "byebug", "~> 11.1.3"
+  spec.add_development_dependency "pg", "~> 1.5.6"
+  spec.add_development_dependency "rubocop", "~> 1.63.4"
+  spec.add_development_dependency "rubocop-packaging"
+  spec.add_development_dependency "rubocop-performance"
+  spec.add_development_dependency "rubocop-rails"
+  spec.add_development_dependency "rubocop-factory_bot", "~> 2.26"
+  spec.add_development_dependency "rubocop-md"
+  spec.add_dependency "rails"
+  spec.add_dependency "rspec-rails"
+  spec.add_dependency "range_operators", "~> 0.1.1"
+
+  # For more information and examples about making a new gem, checkout our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/bin/console

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "active_partition"
+require "active_record"
+require "byebug"
+
+def reload!
+  files = $LOADED_FEATURES.select { |feat| feat =~ /\/active_partition\// }
+  files.each { |file| load file }
+end
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+# Create test model
+class OutgoingEvent < ActiveRecord::Base
+  include ActivePartition::Partitionable
+  self.partitioned_by = "created_at"
+  self.partition_range = 1.day
+
+  # You can choose 1 of the following 2 options
+  self.retention_period = 1.month
+  self.retention_partition_count = 3
+end
+
+OutgoingEvent.establish_connection(ENV["DATABASE_URL"])
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/active_partition/adapters/postgresql_adapter.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module ActivePartition::Adapters
+  class PostgresqlAdapter
+    def initialize(connection, table_name)
+      @connection = connection
+      @table_name = table_name
+    end
+    # Creates a new partition for the table based on the specified time range.
+    #
+    # @param from [Time] The start time of the partition range.
+    # @param to [Time] The end time of the partition range.
+    # @return [Range] The time range of the created partition.
+    def exec_create_partition_by_time_range(partition_name, unix_from, unix_to)
+      sql_from = unix_from.utc.strftime("%Y-%m-%d %H:%M:%S")
+      sql_to = unix_to.utc.strftime("%Y-%m-%d %H:%M:%S")
+
+      @connection.execute <<~SQL
+      CREATE TABLE IF NOT EXISTS #{partition_name}
+        PARTITION OF #{@table_name}
+        FOR VALUES FROM ('#{sql_from}') TO ('#{sql_to}');
+      SQL
+    end
+
+    # Retrieves all supported partition tables for a given table name.
+    #
+    # @return [Array<String>] An array of table names representing the supported partition tables.
+    def get_all_supported_partition_tables
+      table_names_tuples = @connection.execute <<~SQL
+      SELECT relname
+        FROM pg_class c
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+      WHERE nspname = 'public' AND
+              relname LIKE '#{@table_name}_%' AND
+              relkind = 'r'
+      SQL
+
+      table_names = table_names_tuples.map { |tuple| tuple["relname"] }
+      # Filter supported partition names
+      table_names.select { |name| name.match(/#{@table_name}_p_[0-9]{6}_[0-9]{2}_[0-9]{10}_[0-9]{10}/) }
+    end
+
+    # Detaches a partition from the table.
+    #
+    # @param partition_name [String] The name of the partition to detach.
+    # @return [void]
+    def detach_partition(partition_name)
+      @connection.execute <<~SQL
+        ALTER TABLE IF EXISTS #{@table_name} DETACH PARTITION #{partition_name};
+      SQL
+    end
+
+    # Drops a partition table with the given name.
+    #
+    # @param partition_name [String] the name of the partition table to drop
+    # @return [void]
+    def drop_partition(partition_name)
+      @connection.execute <<~SQL
+        DROP TABLE IF EXISTS #{partition_name};
+      SQL
+    end
+  end
+end

data/lib/active_partition/partition_managers/time_range.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+module ActivePartition::PartitionManagers
+  class TimeRange
+    def initialize(partition_adapter, table_name)
+      @partition_adapter = partition_adapter
+      @table_name = table_name
+    end
+
+    # Retrieves the active ranges from the partition adapter.
+    #
+    # The active ranges are cached in an instance variable `@active_ranges` to improve performance.
+    # If the `@active_ranges` variable is `nil`, the method calls the `reload_active_ranges` method
+    # with the result of `@partition_adapter.get_all_supported_partition_tables` as the argument.
+    #
+    # @return [Array] The array of active ranges.
+    def active_ranges
+      @active_ranges ||= reload_active_ranges(@partition_adapter.get_all_supported_partition_tables)
+    end
+
+    # Reloads the active ranges based on the given partition names.
+    #
+    # @param partition_names [Array<String>] An array of partition names.
+    # @return [Array<Range>] An array of Range objects representing the active ranges.
+    def reload_active_ranges(partition_names)
+      @active_ranges = partition_names.map do |partition_name|
+        start_at, end_at = partition_name.split("_").last(2).map { |t| Time.at(t.to_i).utc }
+        (start_at...end_at)
+      end
+    end
+
+    # Checks if the active partitions cover the given value.
+    #
+    # @param value [Time] The value to check if it is covered by the active partitions.
+    # @return [Boolean] Returns true if the value is covered by any of the active partitions, otherwise returns false.
+    def active_partitions_cover?(value)
+      active_ranges.any? { |range| range.cover? value.utc }
+    end
+
+    # Returns the latest coverage time for the partition.
+    #
+    # This method memoizes the latest coverage time by caching the result in an instance variable.
+    # If the latest coverage time has already been calculated, it will be returned from the cache.
+    # Otherwise, it will call the `latest_partition_coverage_time` method to calculate the latest coverage time.
+    #
+    # @return [Time] The latest coverage time for the partition.
+    def latest_coverage_at
+      @latest_coverage_at ||= latest_partition_coverage_time
+    end
+
+    # Prepares a partition for the given partitioned value and period.
+    #
+    # If the active partitions do not cover the partitioned value, a new partition is created.
+    #
+    # @param partitioned_value [Time] The value to be partitioned.
+    # @param period [Integer] The duration of each partition.
+    # @return [void]
+    def prepare_partition(partitioned_value, period)
+      return if active_partitions_cover?(partitioned_value)
+
+      diff = (partitioned_value.utc - latest_coverage_at) / period
+      from_time = latest_coverage_at + (diff.floor * period)
+      to_time = from_time + period
+
+      create_partition(from_time, to_time)
+    end
+
+    # Builds a partition name based on the given time range.
+    #
+    # @param from [DateTime] The start time of the partition range.
+    # @param to [DateTime] The end time of the partition range.
+    # @return [String] The generated partition name.
+    def build_partition_name(from, to)
+      unix_from = from.utc.to_i
+      unix_to = to.utc.to_i
+
+      # It's easier to manage when having readable part in the name
+      readable_from = from.utc.strftime("%y%m%d_%H")
+
+      "#{@table_name}_p_#{readable_from}_#{unix_from}_#{unix_to}"
+    end
+
+    # Creates a new partition for the table based on the specified time range.
+    #
+    # @param from [Time] The start time of the partition range.
+    # @param to [Time] The end time of the partition range.
+    # @return [Range] The time range of the created partition.
+    def create_partition(from, to)
+      from = from.utc
+      to = to.utc
+
+      partition_name = build_partition_name(from, to)
+      puts "create partition #{partition_name} from #{from} to #{to}"
+      @partition_adapter.exec_create_partition_by_time_range(partition_name, from, to)
+
+      reload_active_ranges(@partition_adapter.get_all_supported_partition_tables)
+
+      # rescue ActiveRecord::StatementInvalid => e
+      # byebug
+      # # When overlapping partition, the message will be like this:
+      # # PG::InvalidObjectDefinition: ERROR:  partition "table_name_p_240626_09_1719395833_1719482233" would overlap partition "table_name_p_240627_09_1719481818_1719568218"
+      # # LINE 3:   FOR VALUES FROM ('2024-06-26 09:57:13') TO ('2024-06-27 09
+      # # catchup the floor of the from time to the conflict partition and retry
+      # # handle the floor? what about the ceil?
+      # if e.message.include?("would overlap partition")
+      #   overlapped_partition = e.message.split("would overlap partition").last.split("\n").first.delete('"').strip
+      #   overlapped_from, overlapped_to = overlapped_partition.split("_").last(2).map { |t| Time.at(t.to_i).utc }
+
+      #   return true if (overlapped_from..overlapped_to).cover?(unix_from..unix_to)
+      #   # unix_from < unix_to
+      #   # overlapped_from < overlapped_to
+      #   # if unix_from < overlapped_from
+      #   #   overlapped_from = unix_from
+
+      #   if floor_time > unix_from
+      #     Rails.logger.warn "Retry create partition for #{unix_from} to #{floor_time}"
+      #     create_partition(unix_from, floor_time)
+      #   end
+      # end
+    end
+
+    # Returns the coverage time of the latest partition.
+    #
+    # If there are no supported partition tables, the coverage time will be the beginning of the current hour in UTC.
+    # Otherwise, the coverage time will be extracted from the latest partition table name.
+    #
+    # @return [Time] The coverage time of the latest partition in UTC.
+    def latest_partition_coverage_time
+      partition_tables = @partition_adapter.get_all_supported_partition_tables
+      reload_active_ranges(partition_tables)
+      return Time.current.beginning_of_hour.utc if partition_tables.empty?
+
+      latest_partition_table = partition_tables.sort_by { |p_name| p_name.split("_").last.to_i }.last
+      @latest_coverage_at = Time.at(latest_partition_table.split("_").last.to_i).utc
+      @latest_coverage_at
+    end
+
+    # Creates multiple partitions in the database based on the given period, number, and starting time.
+    #
+    # @param period [ActiveSupport::Duration] The duration of each partition.
+    # @param number [Integer] The number of partitions to create.
+    # @param from [Time] The starting time for creating partitions. If not provided, the current time is used.
+    #
+    # @return [void]
+    def premake(period = 1.month, number = 3, from = nil)
+      new_latest_coverage_time = (from || Time.current).utc + (period * number)
+      current_coverage_time = from || latest_partition_coverage_time
+
+      while current_coverage_time < new_latest_coverage_time
+        create_partition(current_coverage_time, current_coverage_time + period)
+        current_coverage_time += period
+      end
+    end
+
+    # Removes the specified partitions from the database.
+    #
+    # @param prunable_tables [Array<String>] An array of partition names to be removed.
+    # @return [void]
+    def remove_partitions(prunable_tables)
+      table_names = prunable_tables.each do |partition_name|
+        @partition_adapter.detach_partition(partition_name)
+        @partition_adapter.drop_partition(partition_name)
+      end
+
+      reload_active_ranges(@partition_adapter.get_all_supported_partition_tables)
+      table_names
+    end
+
+    # Retains a specified number of partition tables older than a given period.
+    #
+    # @param period [ActiveSupport::Duration] The duration of time to retain partitions.
+    # @param number [Integer] The number of partitions to retain.
+    # @param from [Time] The reference time from which to calculate the retention period.
+    # @return [void]
+    def retain(period = 1.months, number = 12, from = Time.current.utc)
+      prune_time = (from - (period * (number + 1))).utc
+
+      retain_by_time(prune_time)
+    end
+
+    def retain_by_time(prune_time)
+      partition_tables = @partition_adapter.get_all_supported_partition_tables
+      return if partition_tables.empty?
+
+      prunable_tables = partition_tables.select do |name|
+        p_to_time = Time.at(name.split("_").last.to_i).utc
+        p_to_time < prune_time
+      end
+
+      remove_partitions (prunable_tables)
+    end
+
+    def retain_by_partition_count(retain_number)
+      partition_tables = @partition_adapter.get_all_supported_partition_tables
+      nil if partition_tables.empty?
+
+      current_partition_name = build_partition_name(Time.current, Time.current + 1.hour)
+      past_partitions = partition_tables.select { |name| name <= current_partition_name }.sort
+      prunable_partitions = past_partitions[.. -(retain_number + 2)] # -1 of current partition and -1 as syntax
+
+      remove_partitions(prunable_partitions)
+    end
+  end
+end

data/lib/active_partition/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ActivePartition
+  VERSION = "0.1.0"
+end

data/lib/active_partition.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative "active_partition/version"
+require "active_support/concern"
+require "active_support/core_ext/module/delegation"
+require "active_partition/adapters/postgresql_adapter"
+require "active_partition/partition_managers/time_range"
+require "range_operators"
+
+module ActivePartition
+  class Error < StandardError; end
+
+  module Partitionable
+    extend ActiveSupport::Concern
+
+    included do
+      before_create :create_partition_if_needed
+
+      def create_partition_if_needed
+        # get partitioned attribute value
+        partitioned_value = attributes[self.class.partitioned_by.to_s]
+        self.class.prepare_partition(partitioned_value, self.class.partition_range)
+      end
+    end
+
+    # rubocop:disable Metrics
+    class_methods do
+      # The range of each partition. You can change this value over time.
+      # example: 1.month, 2.weeks, 3.hours
+      attr_accessor :partition_range
+      # The column name to partition the table by
+      attr_accessor :partitioned_by
+      # Retains partitions until the specified time [Choose one of retention_period or retention_partition_count]
+      # For example: 1.month (1 month from now), 2.weeks (2 weeks from now), 3.hours (3 hours from now)
+      attr_accessor :retention_period
+      # Retains the specified number of partitions [Choose one of retention_period or retention_partition_count]
+      attr_accessor :retention_partition_count
+
+      def partition_adapter
+        @@partition_adapter ||= ActivePartition::Adapters::PostgresqlAdapter.new(connection, table_name)
+      end
+
+      def partition_manager
+        @@partition_manager ||= case columns_hash[partitioned_by.to_s].type.to_s
+                                when "datetime"
+                                  ActivePartition::PartitionManagers::TimeRange.new(partition_adapter, table_name)
+                                else
+                                  ActivePartition::PartitionManagers::TimeRange.new(partition_adapter, table_name)
+        end
+      end
+
+      def delete_expired_partitions
+        if retention_period && retention_period.is_a?(ActiveSupport::Duration)
+          partition_manager.retain_by_time(retention_period.ago)
+        elsif retention_partition_count
+          partition_manager.retain_by_partition_count(retention_partition_count)
+        end
+      end
+
+      delegate :premake, :latest_partition_coverage_time, to: :partition_manager
+      delegate :retain, :retain_by_time, :retain_by_partition_count, to: :partition_manager
+      delegate :prepare_partition, "active_partitions_cover?", to: :partition_manager
+      delegate :get_all_supported_partition_tables, to: :partition_adapter
+      delegate :drop_partition, to: :partition_adapter
+    end
+  end
+end