partition_gardener 0.3.0

data/docs/schemas/partition_garden.schema.json

@@ -0,0 +1,114 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/amkisko/partition_gardener.rb/main/docs/schemas/partition_garden.schema.json",
+  "title": "Partition Gardener table registration",
+  "description": "Portable fields for registering a partitioned parent table. Ruby lambdas (partition_name_format, partition_definition) are omitted; use layout templates instead.",
+  "type": "object",
+  "required": ["table_name", "layout", "partition_key_column", "conflict_key"],
+  "additionalProperties": false,
+  "properties": {
+    "table_name": {
+      "type": "string",
+      "minLength": 1
+    },
+    "layout": {
+      "type": "string",
+      "enum": [
+        "sliding_window",
+        "rolling_current",
+        "calendar_year",
+        "premake_monthly",
+        "integer_window",
+        "hash_branches"
+      ]
+    },
+    "bucket": {
+      "type": "string",
+      "enum": ["day", "week", "month", "quarter", "year"]
+    },
+    "partition_key_column": {
+      "type": "string",
+      "minLength": 1
+    },
+    "conflict_key": {
+      "type": "array",
+      "items": { "type": "string" },
+      "minItems": 1
+    },
+    "active_days": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 366
+    },
+    "active_weeks": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 104
+    },
+    "active_months": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 36
+    },
+    "active_quarters": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 20
+    },
+    "active_years": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 10
+    },
+    "premake_months": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 24
+    },
+    "split_row_threshold": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "move_batch_size": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "statement_timeout": {
+      "type": "integer",
+      "minimum": 1,
+      "description": "Statement timeout in seconds for maintenance on this table"
+    },
+    "retention_months": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "retention_apply": {
+      "type": "boolean",
+      "description": "When true, detach or drop archive partitions past retention_months; when false or omitted, log would-drop actions only"
+    },
+    "retention_keep_table": {
+      "type": "boolean"
+    },
+    "retention_detach_concurrently": {
+      "type": "boolean"
+    },
+    "hash_modulus": {
+      "type": "integer",
+      "minimum": 2,
+      "maximum": 128
+    },
+    "maintenance_backend": {
+      "type": "string",
+      "enum": ["gardener", "pg_partman", "hybrid_layout_only"]
+    },
+    "incremental_rebalance": {
+      "type": "boolean"
+    },
+    "run_record_enabled": {
+      "type": "boolean"
+    },
+    "analyze_after_rebalance": {
+      "type": "boolean"
+    }
+  }
+}

data/docs/schemas/plan_report.schema.json

@@ -0,0 +1,82 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/amkisko/partition_gardener.rb/main/docs/schemas/plan_report.schema.json",
+  "title": "Partition Gardener plan report",
+  "description": "Dry-run output from PartitionGardener.plan or run!(dry_run: true)",
+  "type": "object",
+  "required": [
+    "schema_version",
+    "table_name",
+    "layout",
+    "changed",
+    "plan_signature",
+    "target_segments",
+    "attached_segments",
+    "operations",
+    "gaps",
+    "hot_buckets"
+  ],
+  "additionalProperties": false,
+  "properties": {
+    "schema_version": {
+      "type": "string",
+      "const": "1.0"
+    },
+    "table_name": { "type": "string" },
+    "layout": { "type": "string" },
+    "changed": { "type": "boolean" },
+    "plan_signature": { "type": "string" },
+    "target_segments": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/segment" }
+    },
+    "attached_segments": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/segment" }
+    },
+    "operations": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/operation" }
+    },
+    "gaps": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/gap" }
+    },
+    "hot_buckets": {
+      "type": "array"
+    }
+  },
+  "$defs": {
+    "segment": {
+      "type": "object",
+      "required": ["name", "range_start", "range_end", "kind"],
+      "properties": {
+        "name": { "type": "string" },
+        "range_start": {},
+        "range_end": {},
+        "kind": { "type": "string" }
+      }
+    },
+    "operation": {
+      "type": "object",
+      "required": ["action"],
+      "properties": {
+        "action": {
+          "type": "string",
+          "enum": ["keep", "create", "reshape", "drop"]
+        },
+        "segment": { "$ref": "#/$defs/segment" },
+        "attached_segment": { "$ref": "#/$defs/segment" }
+      }
+    },
+    "gap": {
+      "type": "object",
+      "required": ["message"],
+      "properties": {
+        "range_start": {},
+        "range_end": {},
+        "message": { "type": "string" }
+      }
+    }
+  }
+}

data/docs/tooling_split.md

@@ -0,0 +1,79 @@
+# Tooling split: DDL creation, runtime maintenance, and extension maintenance
+
+Partition work splits into three layers. Each layer has a different owner and schedule. Mixing owners on the same parent table without an explicit contract causes overlapping bounds, double drops, and rows stuck in the default partition.
+
+## pg_party — creation (DDL)
+
+Use during migrations and cutover.
+
+- Create partitioned parent and first children
+- Attach indexes across children
+- Model helpers for partition DDL
+
+pg_party does not replace nightly maintenance. After cutover, hand the table to runtime maintenance.
+
+## pg_partman — extension runtime maintenance (premake and retention)
+
+Use when the hosting environment allows the extension and the table is a plain time or id series.
+
+- Premake upcoming intervals on a fixed cadence
+- Retention via detach or drop
+- Background worker or `run_maintenance_proc`
+
+partman fits tables that do not need three-area layout, heat splits inside current, or default-last drain logic.
+
+Register such tables with `maintenance_backend: :pg_partman` so Partition Gardener skips them.
+
+## Partition Gardener — application runtime maintenance (layout)
+
+Use for runtime maintenance when layout is more than fixed premake.
+
+- Sliding window archive, current, and future
+- Heat-driven splits inside current
+- Mandatory default drain last
+- Incremental tail rebalance and gap detection
+- Hot-switch cutover helper
+
+Gardener is the default `maintenance_backend: :gardener`.
+
+## Pick one maintainer per table
+
+New date-keyed OLTP table — create with pg_party or SQL; maintain with Gardener sliding window.
+
+Plain monthly series when the extension is OK — create with pg_party or SQL; maintain with partman only (`maintenance_backend: :pg_partman`).
+
+Migrating from premake-only cron — keep existing DDL; use Gardener after `premake_monthly` bridge, then upgrade template.
+
+partman premake plus custom tail layout — create with pg_party or SQL; maintain with `maintenance_backend: :hybrid_layout_only` (gardener layout only; partman premake).
+
+## Hybrid mode
+
+`maintenance_backend: :hybrid_layout_only` means:
+
+- pg_partman owns premake and simple retention on `partman.part_config`
+- Gardener runs tail rebalance, default drain, and gap repair only
+- Gardener must not create the same monthly bounds partman already premakes
+
+On register, Gardener probes `partman.part_config` and warns when:
+
+- `maintenance_backend: :gardener` but partman also lists the parent
+- `maintenance_backend: :pg_partman` but partman has no row
+- `hybrid_layout_only` but partman has no row
+
+## Legacy premake template
+
+`Templates.premake_monthly` is a stepping stone from cron premake jobs.
+
+- Ensures current month through `premake_months` ahead exist
+- Drains default
+- Does not run sliding-window tail rebalance
+
+Migrate to `sliding_window_monthly` when default stays empty and horizon warnings clear.
+
+## Related
+
+- [decision_flow.md](decision_flow.md) — when to partition and which layout
+- [configuration.md](configuration.md) — `maintenance_backend` and per-table registry
+- [background_job.md](background_job.md) — schedule gardener runs
+- [cutover.md](cutover.md) — creation through hot switch
+- [related_postgres_tooling.md](related_postgres_tooling.md) — ankane tools; pgslice hot-switch detail in [cutover.md](cutover.md#lessons-from-pgslice)

data/exe/partition_gardener

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "partition_gardener"
+require "partition_gardener/cli"
+
+PartitionGardener::CLI.start

data/lib/partition_gardener/active_record_run_record_store.rb

@@ -0,0 +1 @@
+require_relative "sql_run_record_store"

data/lib/partition_gardener/advisory_lock.rb

@@ -0,0 +1,48 @@
+module PartitionGardener
+  module AdvisoryLock
+    LOCK_NAMESPACE = "partition_gardener"
+
+    module_function
+
+    def with_table_lock(table_name, &block)
+      case PartitionGardener.configuration.advisory_lock_mode
+      when :transaction
+        with_transaction_lock(table_name, &block)
+      else
+        with_session_lock(table_name, &block)
+      end
+    end
+
+    def with_session_lock(table_name)
+      connection = Connection.connection
+      lock_sql = lock_expression(connection, table_name)
+      acquired = false
+      acquired = connection.execute("SELECT pg_try_advisory_lock(#{lock_sql})").first.values.first
+      acquired = acquired == true || acquired == "t"
+      raise LockNotAcquired, "session advisory lock not acquired for #{table_name}" unless acquired
+
+      yield
+    ensure
+      connection.execute("SELECT pg_advisory_unlock(#{lock_sql})") if connection && lock_sql && acquired
+    end
+
+    def with_transaction_lock(table_name)
+      connection = Connection.connection
+      lock_sql = lock_expression(connection, table_name)
+
+      connection.transaction do
+        acquired = connection.execute("SELECT pg_try_advisory_xact_lock(#{lock_sql})").first.values.first
+        acquired = acquired == true || acquired == "t"
+        raise LockNotAcquired, "transaction advisory lock not acquired for #{table_name}" unless acquired
+
+        yield
+      end
+    end
+
+    def lock_expression(connection, table_name)
+      namespace = connection.quote(LOCK_NAMESPACE)
+      table = connection.quote(table_name)
+      "hashtext(#{namespace}), hashtext(#{table})"
+    end
+  end
+end

data/lib/partition_gardener/archive_retention.rb

@@ -0,0 +1,88 @@
+module PartitionGardener
+  class ArchiveRetention
+    include Naming
+
+    def initialize(config, executor: nil, job_class_name: "PartitionGardener")
+      @config = config
+      @executor = executor || Executor.for_config(config)
+      @job_class_name = job_class_name
+    end
+
+    def apply!(dry_run: nil)
+      retention_months = @config[:retention_months]
+      return 0 unless retention_months
+
+      apply_retention = if dry_run.nil?
+        @config.fetch(:retention_apply, false)
+      else
+        !dry_run
+      end
+
+      strategy = Strategy.for(@config)
+      return 0 unless strategy.is_a?(Strategy::DateRange)
+
+      table_name = @config[:table_name]
+      cutoff = DateCalendar.add_months(PartitionGardener.configuration.today, -retention_months)
+      dropped = 0
+      managed_tail_names = managed_tail_name_set(strategy)
+
+      Connection.attached_partitions(table_name).each do |partition|
+        next if partition.default
+        next if skip_retention_partition?(strategy, partition.name, managed_tail_names)
+
+        bucket = strategy.archive_bucket_from_partition_name(partition.name)
+        next unless bucket
+        next if bucket >= DateCalendar.beginning_of_month(cutoff)
+
+        if apply_retention
+          @executor.detach_partition(table_name, partition.name, concurrently: detach_concurrently?)
+          @executor.drop_table(partition.name) unless @config.fetch(:retention_keep_table, false)
+
+          PartitionGardener.configuration.notify(
+            "[PartitionGardener] Dropped archive partition #{partition.name} (retention #{retention_months} months)",
+            context: {
+              table_name: table_name,
+              partition_name: partition.name,
+              retention_months: retention_months,
+              job: @job_class_name
+            }
+          )
+          dropped += 1
+          next
+        end
+
+        PartitionGardener.configuration.notify(
+          "[PartitionGardener] Would drop archive partition #{partition.name} (bucket #{bucket}, retention #{retention_months} months)",
+          context: {
+            table_name: table_name,
+            partition_name: partition.name,
+            bucket: bucket,
+            dry_run: true
+          }
+        )
+        dropped += 1
+      end
+
+      dropped
+    end
+
+    private
+
+    def detach_concurrently?
+      @config.fetch(:retention_detach_concurrently, PartitionGardener.configuration.retention_detach_concurrently)
+    end
+
+    def managed_tail_name_set(strategy)
+      Set.new(strategy.managed_tail_partition_names)
+    rescue NoMethodError
+      Set.new
+    end
+
+    def skip_retention_partition?(strategy, partition_name, managed_tail_names)
+      (strategy.respond_to?(:tail_slot_name?) && strategy.tail_slot_name?(partition_name)) ||
+        managed_tail_names.include?(partition_name)
+    rescue NoMethodError
+      false
+    end
+  end
+end

data/lib/partition_gardener/audit.rb

@@ -0,0 +1,93 @@
+module PartitionGardener
+  class Audit
+    SCHEMA_VERSION = "1.0"
+    HORIZON_WARNING_DAYS = 30
+
+    AuditResult = Data.define(
+      :table_name,
+      :partitioned,
+      :default_row_count,
+      :attached_child_count,
+      :horizon_days,
+      :gaps,
+      :warnings
+    )
+
+    def self.call(table_name, config: Registry.find_by_table_name(table_name))
+      new(table_name, config: config).call
+    end
+
+    def initialize(table_name, config: nil)
+      @table_name = table_name
+      @config = config || Registry.find_by_table_name(table_name)
+    end
+
+    def call
+      unless Connection.table_is_partitioned?(@table_name)
+        return AuditResult.new(
+          table_name: @table_name,
+          partitioned: false,
+          default_row_count: 0,
+          attached_child_count: 0,
+          horizon_days: nil,
+          gaps: [],
+          warnings: ["#{@table_name} is not a partitioned table"]
+        )
+      end
+
+      default_name = Naming.default_partition_name(@table_name)
+      default_row_count = if Connection.partition_exists?(default_name)
+        Connection.count_rows_in_partition_table(default_name)
+      else
+        0
+      end
+
+      partitions = Connection.attached_partitions(@table_name)
+      warnings = []
+      warnings << "default partition #{default_name} has #{default_row_count} rows" if default_row_count.positive?
+      warnings << "default partition #{default_name} is missing" unless Connection.partition_exists?(default_name)
+
+      horizon_days = compute_horizon_days(partitions)
+      if horizon_days && horizon_days < HORIZON_WARNING_DAYS
+        warnings << "partition horizon is #{horizon_days} days ahead (below #{HORIZON_WARNING_DAYS})"
+      end
+
+      if partitions.size > 200
+        warnings << "attached child count is #{partitions.size} (high catalog pressure)"
+      end
+
+      gaps = GapDetection.call(@table_name, config: @config)
+      gaps.each do |gap|
+        warnings << "partition gap: #{gap.message}"
+      end
+
+      AuditResult.new(
+        table_name: @table_name,
+        partitioned: true,
+        default_row_count: default_row_count,
+        attached_child_count: partitions.size,
+        horizon_days: horizon_days,
+        gaps: gaps,
+        warnings: warnings
+      )
+    end
+
+    private
+
+    def compute_horizon_days(partitions)
+      today = PartitionGardener.configuration.today
+      finite_ends = partitions.filter_map do |partition|
+        next if partition.default
+        next if partition.range_end == :max
+        next unless partition.range_end.is_a?(Date)
+
+        partition.range_end
+      end
+
+      return nil if finite_ends.empty?
+
+      max_end = finite_ends.max
+      (max_end - today).to_i
+    end
+  end
+end

data/lib/partition_gardener/blank.rb

@@ -0,0 +1,13 @@
+module PartitionGardener
+  module Blank
+    module_function
+
+    def blank?(value)
+      value.nil? || (value.respond_to?(:empty?) && value.empty?)
+    end
+
+    def present?(value)
+      !blank?(value)
+    end
+  end
+end