pgslice 0.4.4 → 0.4.5

data/lib/pgslice/cli/analyze.rb

@@ -0,0 +1,13 @@
+module PgSlice
+  class CLI
+    desc "analyze TABLE", "Analyze tables"
+    option :swapped, type: :boolean, default: false, desc: "Use swapped table"
+    def analyze(table)
+      table = create_table(table)
+      parent_table = options[:swapped] ? table : table.intermediate_table
+
+      analyze_list = parent_table.partitions + [parent_table]
+      run_queries_without_transaction(analyze_list.map { |t| "ANALYZE VERBOSE #{quote_table(t)};" })
+    end
+  end
+end

data/lib/pgslice/cli/fill.rb

@@ -0,0 +1,103 @@
+module PgSlice
+  class CLI
+    desc "fill TABLE", "Fill the partitions in batches"
+    option :batch_size, type: :numeric, default: 10000, desc: "Batch size"
+    option :swapped, type: :boolean, default: false, desc: "Use swapped table"
+    option :source_table, desc: "Source table"
+    option :dest_table, desc: "Destination table"
+    option :start, type: :numeric, desc: "Primary key to start"
+    option :where, desc: "Conditions to filter"
+    option :sleep, type: :numeric, desc: "Seconds to sleep between batches"
+    def fill(table)
+      table = create_table(table)
+      source_table = create_table(options[:source_table]) if options[:source_table]
+      dest_table = create_table(options[:dest_table]) if options[:dest_table]
+
+      if options[:swapped]
+        source_table ||= table.retired_table
+        dest_table ||= table
+      else
+        source_table ||= table
+        dest_table ||= table.intermediate_table
+      end
+
+      assert_table(source_table)
+      assert_table(dest_table)
+
+      period, field, cast, _, declarative, _ = dest_table.fetch_settings(table.trigger_name)
+
+      if period
+        name_format = self.name_format(period)
+
+        partitions = dest_table.partitions
+        if partitions.any?
+          starting_time = partition_date(partitions.first, name_format)
+          ending_time = advance_date(partition_date(partitions.last, name_format), period, 1)
+        end
+      end
+
+      schema_table = period && declarative ? partitions.last : table
+
+      primary_key = schema_table.primary_key[0]
+      abort "No primary key" unless primary_key
+
+      max_source_id = nil
+      begin
+        max_source_id = source_table.max_id(primary_key)
+      rescue PG::UndefinedFunction
+        abort "Only numeric primary keys are supported"
+      end
+
+      max_dest_id =
+        if options[:start]
+          options[:start]
+        elsif options[:swapped]
+          dest_table.max_id(primary_key, where: options[:where], below: max_source_id)
+        else
+          dest_table.max_id(primary_key, where: options[:where])
+        end
+
+      if max_dest_id == 0 && !options[:swapped]
+        min_source_id = source_table.min_id(primary_key, field, cast, starting_time, options[:where])
+        max_dest_id = min_source_id - 1 if min_source_id
+      end
+
+      starting_id = max_dest_id
+      fields = source_table.columns.map { |c| quote_ident(c) }.join(", ")
+      batch_size = options[:batch_size]
+
+      i = 1
+      batch_count = ((max_source_id - starting_id) / batch_size.to_f).ceil
+
+      if batch_count == 0
+        log_sql "/* nothing to fill */"
+      end
+
+      while starting_id < max_source_id
+        where = "#{quote_ident(primary_key)} > #{starting_id} AND #{quote_ident(primary_key)} <= #{starting_id + batch_size}"
+        if starting_time
+          where << " AND #{quote_ident(field)} >= #{sql_date(starting_time, cast)} AND #{quote_ident(field)} < #{sql_date(ending_time, cast)}"
+        end
+        if options[:where]
+          where << " AND #{options[:where]}"
+        end
+
+        query = <<-SQL
+/* #{i} of #{batch_count} */
+INSERT INTO #{quote_table(dest_table)} (#{fields})
+    SELECT #{fields} FROM #{quote_table(source_table)}
+    WHERE #{where}
+        SQL
+
+        run_query(query)
+
+        starting_id += batch_size
+        i += 1
+
+        if options[:sleep] && starting_id <= max_source_id
+          sleep(options[:sleep])
+        end
+      end
+    end
+  end
+end

data/lib/pgslice/cli/prep.rb

@@ -0,0 +1,97 @@
+module PgSlice
+  class CLI
+    desc "prep TABLE [COLUMN] [PERIOD]", "Create an intermediate table for partitioning"
+    option :partition, type: :boolean, default: true, desc: "Partition the table"
+    option :trigger_based, type: :boolean, default: false, desc: "Use trigger-based partitioning"
+    def prep(table, column=nil, period=nil)
+      table = create_table(table)
+      intermediate_table = table.intermediate_table
+      trigger_name = table.trigger_name
+
+      unless options[:partition]
+        abort "Usage: \"pgslice prep TABLE --no-partition\"" if column || period
+        abort "Can't use --trigger-based and --no-partition" if options[:trigger_based]
+      end
+      assert_table(table)
+      assert_no_table(intermediate_table)
+
+      if options[:partition]
+        abort "Usage: \"pgslice prep TABLE COLUMN PERIOD\"" if !(column && period)
+        abort "Column not found: #{column}" unless table.columns.include?(column)
+        abort "Invalid period: #{period}" unless SQL_FORMAT[period.to_sym]
+      end
+
+      queries = []
+
+      # version summary
+      # 1. trigger-based
+      # 2. declarative, with indexes and foreign keys on child tables
+      # 3. declarative, with indexes and foreign keys on parent table
+      version =
+        if options[:trigger_based] || server_version_num < 100000
+          1
+        elsif server_version_num < 110000
+          2
+        else
+          3
+        end
+
+      declarative = version > 1
+
+      if declarative && options[:partition]
+        queries << <<-SQL
+CREATE TABLE #{quote_table(intermediate_table)} (LIKE #{quote_table(table)} INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE INCLUDING COMMENTS) PARTITION BY RANGE (#{quote_ident(column)});
+        SQL
+
+        if version == 3
+          index_defs = table.index_defs
+          index_defs.each do |index_def|
+            queries << make_index_def(index_def, intermediate_table)
+          end
+
+          table.foreign_keys.each do |fk_def|
+            queries << make_fk_def(fk_def, intermediate_table)
+          end
+        end
+
+        # add comment
+        cast = table.column_cast(column)
+        queries << <<-SQL
+COMMENT ON TABLE #{quote_table(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast},version:#{version}';
+        SQL
+      else
+        queries << <<-SQL
+CREATE TABLE #{quote_table(intermediate_table)} (LIKE #{quote_table(table)} INCLUDING ALL);
+        SQL
+
+        table.foreign_keys.each do |fk_def|
+          queries << make_fk_def(fk_def, intermediate_table)
+        end
+      end
+
+      if options[:partition] && !declarative
+        queries << <<-SQL
+CREATE FUNCTION #{quote_ident(trigger_name)}()
+    RETURNS trigger AS $$
+    BEGIN
+        RAISE EXCEPTION 'Create partitions first.';
+    END;
+    $$ LANGUAGE plpgsql;
+        SQL
+
+        queries << <<-SQL
+CREATE TRIGGER #{quote_ident(trigger_name)}
+    BEFORE INSERT ON #{quote_table(intermediate_table)}
+    FOR EACH ROW EXECUTE PROCEDURE #{quote_ident(trigger_name)}();
+        SQL
+
+        cast = table.column_cast(column)
+        queries << <<-SQL
+COMMENT ON TRIGGER #{quote_ident(trigger_name)} ON #{quote_table(intermediate_table)} is 'column:#{column},period:#{period},cast:#{cast}';
+        SQL
+      end
+
+      run_queries(queries)
+    end
+  end
+end

data/lib/pgslice/cli/swap.rb

@@ -0,0 +1,28 @@
+module PgSlice
+  class CLI
+    desc "swap TABLE", "Swap the intermediate table with the original table"
+    option :lock_timeout, default: "5s", desc: "Lock timeout"
+    def swap(table)
+      table = create_table(table)
+      intermediate_table = table.intermediate_table
+      retired_table = table.retired_table
+
+      assert_table(table)
+      assert_table(intermediate_table)
+      assert_no_table(retired_table)
+
+      queries = [
+        "ALTER TABLE #{quote_table(table)} RENAME TO #{quote_no_schema(retired_table)};",
+        "ALTER TABLE #{quote_table(intermediate_table)} RENAME TO #{quote_no_schema(table)};"
+      ]
+
+      table.sequences.each do |sequence|
+        queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_table(table)}.#{quote_ident(sequence["related_column"])};"
+      end
+
+      queries.unshift("SET LOCAL lock_timeout = '#{options[:lock_timeout]}';") if server_version_num >= 90300
+
+      run_queries(queries)
+    end
+  end
+end

data/lib/pgslice/cli/unprep.rb

@@ -0,0 +1,18 @@
+module PgSlice
+  class CLI
+    desc "unprep TABLE", "Undo prep"
+    def unprep(table)
+      table = create_table(table)
+      intermediate_table = table.intermediate_table
+      trigger_name = table.trigger_name
+
+      assert_table(intermediate_table)
+
+      queries = [
+        "DROP TABLE #{quote_table(intermediate_table)} CASCADE;",
+        "DROP FUNCTION IF EXISTS #{quote_ident(trigger_name)}();"
+      ]
+      run_queries(queries)
+    end
+  end
+end

data/lib/pgslice/cli/unswap.rb

@@ -0,0 +1,25 @@
+module PgSlice
+  class CLI
+    desc "unswap TABLE", "Undo swap"
+    def unswap(table)
+      table = create_table(table)
+      intermediate_table = table.intermediate_table
+      retired_table = table.retired_table
+
+      assert_table(table)
+      assert_table(retired_table)
+      assert_no_table(intermediate_table)
+
+      queries = [
+        "ALTER TABLE #{quote_table(table)} RENAME TO #{quote_no_schema(intermediate_table)};",
+        "ALTER TABLE #{quote_table(retired_table)} RENAME TO #{quote_no_schema(table)};"
+      ]
+
+      table.sequences.each do |sequence|
+        queries << "ALTER SEQUENCE #{quote_ident(sequence["sequence_name"])} OWNED BY #{quote_table(table)}.#{quote_ident(sequence["related_column"])};"
+      end
+
+      run_queries(queries)
+    end
+  end
+end

data/lib/pgslice/helpers.rb

@@ -0,0 +1,176 @@
+module PgSlice
+  module Helpers
+    SQL_FORMAT = {
+      day: "YYYYMMDD",
+      month: "YYYYMM",
+      year: "YYYY"
+    }
+
+    protected
+
+    # output
+
+    def log(message = nil)
+      error message
+    end
+
+    def log_sql(message = nil)
+      say message
+    end
+
+    def abort(message)
+      raise Thor::Error, message
+    end
+
+    # database connection
+
+    def connection
+      @connection ||= begin
+        url = options[:url] || ENV["PGSLICE_URL"]
+        abort "Set PGSLICE_URL or use the --url option" unless url
+
+        uri = URI.parse(url)
+        params = CGI.parse(uri.query.to_s)
+        # remove schema
+        @schema = Array(params.delete("schema") || "public")[0]
+        uri.query = URI.encode_www_form(params)
+
+        ENV["PGCONNECT_TIMEOUT"] ||= "1"
+        PG::Connection.new(uri.to_s)
+      end
+    rescue PG::ConnectionBad => e
+      abort e.message
+    rescue URI::InvalidURIError
+      abort "Invalid url"
+    end
+
+    def schema
+      connection # ensure called first
+      @schema
+    end
+
+    def execute(query, params = [])
+      connection.exec_params(query, params).to_a
+    end
+
+    def run_queries(queries)
+      connection.transaction do
+        execute("SET LOCAL client_min_messages TO warning") unless options[:dry_run]
+        log_sql "BEGIN;"
+        log_sql
+        run_queries_without_transaction(queries)
+        log_sql "COMMIT;"
+      end
+    end
+
+    def run_query(query)
+      log_sql query
+      unless options[:dry_run]
+        begin
+          execute(query)
+        rescue PG::ServerError => e
+          abort("#{e.class.name}: #{e.message}")
+        end
+      end
+      log_sql
+    end
+
+    def run_queries_without_transaction(queries)
+      queries.each do |query|
+        run_query(query)
+      end
+    end
+
+    def server_version_num
+      execute("SHOW server_version_num")[0]["server_version_num"].to_i
+    end
+
+    # helpers
+
+    def sql_date(time, cast, add_cast = true)
+      if cast == "timestamptz"
+        fmt = "%Y-%m-%d %H:%M:%S UTC"
+      else
+        fmt = "%Y-%m-%d"
+      end
+      str = "'#{time.strftime(fmt)}'"
+      add_cast ? "#{str}::#{cast}" : str
+    end
+
+    def name_format(period)
+      case period.to_sym
+      when :day
+        "%Y%m%d"
+      when :month
+        "%Y%m"
+      else
+        "%Y"
+      end
+    end
+
+    def partition_date(partition, name_format)
+      DateTime.strptime(partition.name.split("_").last, name_format)
+    end
+
+    def round_date(date, period)
+      date = date.to_date
+      case period.to_sym
+      when :day
+        date
+      when :month
+        Date.new(date.year, date.month)
+      else
+        Date.new(date.year)
+      end
+    end
+
+    def assert_table(table)
+      abort "Table not found: #{table}" unless table.exists?
+    end
+
+    def assert_no_table(table)
+      abort "Table already exists: #{table}" if table.exists?
+    end
+
+    def advance_date(date, period, count = 1)
+      date = date.to_date
+      case period.to_sym
+      when :day
+        date.next_day(count)
+      when :month
+        date.next_month(count)
+      else
+        date.next_year(count)
+      end
+    end
+
+    def quote_ident(value)
+      PG::Connection.quote_ident(value)
+    end
+
+    def quote_table(table)
+      table.quote_table
+    end
+
+    def quote_no_schema(table)
+      quote_ident(table.name)
+    end
+
+    def create_table(name)
+      if name.include?(".")
+        schema, name = name.split(".", 2)
+      else
+        schema = self.schema
+      end
+      Table.new(schema, name)
+    end
+
+    def make_index_def(index_def, table)
+      index_def.sub(/ ON \S+ USING /, " ON #{quote_table(table)} USING ").sub(/ INDEX .+ ON /, " INDEX ON ") + ";"
+    end
+
+    def make_fk_def(fk_def, table)
+      "ALTER TABLE #{quote_table(table)} ADD #{fk_def};"
+    end
+  end
+end