pgslice 0.4.2 → 0.4.7

data/lib/pgslice/helpers.rb

@@ -0,0 +1,180 @@
+module PgSlice
+  module Helpers
+    SQL_FORMAT = {
+      day: "YYYYMMDD",
+      month: "YYYYMM",
+      year: "YYYY"
+    }
+
+    protected
+
+    # output
+
+    def log(message = nil)
+      error message
+    end
+
+    def log_sql(message = nil)
+      say message
+    end
+
+    def abort(message)
+      raise Thor::Error, message
+    end
+
+    # database connection
+
+    def connection
+      @connection ||= begin
+        url = options[:url] || ENV["PGSLICE_URL"]
+        abort "Set PGSLICE_URL or use the --url option" unless url
+
+        uri = URI.parse(url)
+        params = CGI.parse(uri.query.to_s)
+        # remove schema
+        @schema = Array(params.delete("schema") || "public")[0]
+        uri.query = URI.encode_www_form(params)
+
+        ENV["PGCONNECT_TIMEOUT"] ||= "1"
+        conn = PG::Connection.new(uri.to_s)
+        conn.set_notice_processor do |message|
+          say message
+        end
+        conn
+      end
+    rescue PG::ConnectionBad => e
+      abort e.message
+    rescue URI::InvalidURIError
+      abort "Invalid url"
+    end
+
+    def schema
+      connection # ensure called first
+      @schema
+    end
+
+    def execute(query, params = [])
+      connection.exec_params(query, params).to_a
+    end
+
+    def run_queries(queries)
+      connection.transaction do
+        execute("SET LOCAL client_min_messages TO warning") unless options[:dry_run]
+        log_sql "BEGIN;"
+        log_sql
+        run_queries_without_transaction(queries)
+        log_sql "COMMIT;"
+      end
+    end
+
+    def run_query(query)
+      log_sql query
+      unless options[:dry_run]
+        begin
+          execute(query)
+        rescue PG::ServerError => e
+          abort("#{e.class.name}: #{e.message}")
+        end
+      end
+      log_sql
+    end
+
+    def run_queries_without_transaction(queries)
+      queries.each do |query|
+        run_query(query)
+      end
+    end
+
+    def server_version_num
+      execute("SHOW server_version_num")[0]["server_version_num"].to_i
+    end
+
+    # helpers
+
+    def sql_date(time, cast, add_cast = true)
+      if cast == "timestamptz"
+        fmt = "%Y-%m-%d %H:%M:%S UTC"
+      else
+        fmt = "%Y-%m-%d"
+      end
+      str = "'#{time.strftime(fmt)}'"
+      add_cast ? "#{str}::#{cast}" : str
+    end
+
+    def name_format(period)
+      case period.to_sym
+      when :day
+        "%Y%m%d"
+      when :month
+        "%Y%m"
+      else
+        "%Y"
+      end
+    end
+
+    def partition_date(partition, name_format)
+      DateTime.strptime(partition.name.split("_").last, name_format)
+    end
+
+    def round_date(date, period)
+      date = date.to_date
+      case period.to_sym
+      when :day
+        date
+      when :month
+        Date.new(date.year, date.month)
+      else
+        Date.new(date.year)
+      end
+    end
+
+    def assert_table(table)
+      abort "Table not found: #{table}" unless table.exists?
+    end
+
+    def assert_no_table(table)
+      abort "Table already exists: #{table}" if table.exists?
+    end
+
+    def advance_date(date, period, count = 1)
+      date = date.to_date
+      case period.to_sym
+      when :day
+        date.next_day(count)
+      when :month
+        date.next_month(count)
+      else
+        date.next_year(count)
+      end
+    end
+
+    def quote_ident(value)
+      PG::Connection.quote_ident(value)
+    end
+
+    def quote_table(table)
+      table.quote_table
+    end
+
+    def quote_no_schema(table)
+      quote_ident(table.name)
+    end
+
+    def create_table(name)
+      if name.include?(".")
+        schema, name = name.split(".", 2)
+      else
+        schema = self.schema
+      end
+      Table.new(schema, name)
+    end
+
+    def make_index_def(index_def, table)
+      index_def.sub(/ ON \S+ USING /, " ON #{quote_table(table)} USING ").sub(/ INDEX .+ ON /, " INDEX ON ") + ";"
+    end
+
+    def make_fk_def(fk_def, table)
+      "ALTER TABLE #{quote_table(table)} ADD #{fk_def};"
+    end
+  end
+end

data/lib/pgslice/table.rb

@@ -0,0 +1,199 @@
+module PgSlice
+  class Table
+    attr_reader :schema, :name
+
+    def initialize(schema, name)
+      @schema = schema
+      @name = name
+    end
+
+    def to_s
+      [schema, name].join(".")
+    end
+
+    def exists?
+      execute("SELECT COUNT(*) FROM pg_catalog.pg_tables WHERE schemaname = $1 AND tablename = $2", [schema, name]).first["count"].to_i > 0
+    end
+
+    def columns
+      execute("SELECT column_name FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2", [schema, name]).map{ |r| r["column_name"] }
+    end
+
+    # http://www.dbforums.com/showthread.php?1667561-How-to-list-sequences-and-the-columns-by-SQL
+    def sequences
+      query = <<-SQL
+        SELECT
+          a.attname AS related_column,
+          n.nspname AS sequence_schema,
+          s.relname AS sequence_name
+        FROM pg_class s
+          INNER JOIN pg_depend d ON d.objid = s.oid
+          INNER JOIN pg_class t ON d.objid = s.oid AND d.refobjid = t.oid
+          INNER JOIN pg_attribute a ON (d.refobjid, d.refobjsubid) = (a.attrelid, a.attnum)
+          INNER JOIN pg_namespace n ON n.oid = s.relnamespace
+          INNER JOIN pg_namespace nt ON nt.oid = t.relnamespace
+        WHERE s.relkind = 'S'
+          AND nt.nspname = $1
+          AND t.relname = $2
+        ORDER BY s.relname ASC
+      SQL
+      execute(query, [schema, name])
+    end
+
+    def foreign_keys
+      execute("SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid = #{regclass} AND contype ='f'").map { |r| r["pg_get_constraintdef"] }
+    end
+
+    # https://stackoverflow.com/a/20537829
+    # TODO can simplify with array_position in Postgres 9.5+
+    def primary_key
+      query = <<-SQL
+        SELECT
+          pg_attribute.attname,
+          format_type(pg_attribute.atttypid, pg_attribute.atttypmod),
+          pg_attribute.attnum,
+          pg_index.indkey
+        FROM
+          pg_index, pg_class, pg_attribute, pg_namespace
+        WHERE
+          nspname = $1 AND
+          relname = $2 AND
+          indrelid = pg_class.oid AND
+          pg_class.relnamespace = pg_namespace.oid AND
+          pg_attribute.attrelid = pg_class.oid AND
+          pg_attribute.attnum = any(pg_index.indkey) AND
+          indisprimary
+      SQL
+      rows = execute(query, [schema, name])
+      rows.sort_by { |r| r["indkey"].split(" ").index(r["attnum"]) }.map { |r| r["attname"] }
+    end
+
+    def index_defs
+      execute("SELECT pg_get_indexdef(indexrelid) FROM pg_index WHERE indrelid = #{regclass} AND indisprimary = 'f'").map { |r| r["pg_get_indexdef"] }
+    end
+
+    def quote_table
+      [quote_ident(schema), quote_ident(name)].join(".")
+    end
+
+    def intermediate_table
+      self.class.new(schema, "#{name}_intermediate")
+    end
+
+    def retired_table
+      self.class.new(schema, "#{name}_retired")
+    end
+
+    def trigger_name
+      "#{name}_insert_trigger"
+    end
+
+    def column_cast(column)
+      data_type = execute("SELECT data_type FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 AND column_name = $3", [schema, name, column])[0]["data_type"]
+      data_type == "timestamp with time zone" ? "timestamptz" : "date"
+    end
+
+    def max_id(primary_key, below: nil, where: nil)
+      query = "SELECT MAX(#{quote_ident(primary_key)}) FROM #{quote_table}"
+      conditions = []
+      conditions << "#{quote_ident(primary_key)} <= #{below}" if below
+      conditions << where if where
+      query << " WHERE #{conditions.join(" AND ")}" if conditions.any?
+      execute(query)[0]["max"].to_i
+    end
+
+    def min_id(primary_key, column, cast, starting_time, where)
+      query = "SELECT MIN(#{quote_ident(primary_key)}) FROM #{quote_table}"
+      conditions = []
+      conditions << "#{quote_ident(column)} >= #{sql_date(starting_time, cast)}" if starting_time
+      conditions << where if where
+      query << " WHERE #{conditions.join(" AND ")}" if conditions.any?
+      (execute(query)[0]["min"] || 1).to_i
+    end
+
+    # ensure this returns partitions in the correct order
+    def partitions
+      query = <<-SQL
+        SELECT
+          nmsp_child.nspname  AS schema,
+          child.relname       AS name
+        FROM pg_inherits
+          JOIN pg_class parent            ON pg_inherits.inhparent = parent.oid
+          JOIN pg_class child             ON pg_inherits.inhrelid   = child.oid
+          JOIN pg_namespace nmsp_parent   ON nmsp_parent.oid  = parent.relnamespace
+          JOIN pg_namespace nmsp_child    ON nmsp_child.oid   = child.relnamespace
+        WHERE
+          nmsp_parent.nspname = $1 AND
+          parent.relname = $2
+        ORDER BY child.relname ASC
+      SQL
+      execute(query, [schema, name]).map { |r| Table.new(r["schema"], r["name"]) }
+    end
+
+    def fetch_comment
+      execute("SELECT obj_description(#{regclass}) AS comment")[0]
+    end
+
+    def fetch_trigger(trigger_name)
+      execute("SELECT obj_description(oid, 'pg_trigger') AS comment FROM pg_trigger WHERE tgname = $1 AND tgrelid = #{regclass}", [trigger_name])[0]
+    end
+
+    # legacy
+    def fetch_settings(trigger_name)
+      needs_comment = false
+      trigger_comment = fetch_trigger(trigger_name)
+      comment = trigger_comment || fetch_comment
+      if comment
+        field, period, cast, version = comment["comment"].split(",").map { |v| v.split(":").last } rescue []
+        version = version.to_i if version
+      end
+
+      unless period
+        needs_comment = true
+        function_def = execute("SELECT pg_get_functiondef(oid) FROM pg_proc WHERE proname = $1", [trigger_name])[0]
+        return [] unless function_def
+        function_def = function_def["pg_get_functiondef"]
+        sql_format = Helpers::SQL_FORMAT.find { |_, f| function_def.include?("'#{f}'") }
+        return [] unless sql_format
+        period = sql_format[0]
+        field = /to_char\(NEW\.(\w+),/.match(function_def)[1]
+      end
+
+      # backwards compatibility with 0.2.3 and earlier (pre-timestamptz support)
+      unless cast
+        cast = "date"
+        # update comment to explicitly define cast
+        needs_comment = true
+      end
+
+      version ||= trigger_comment ? 1 : 2
+      declarative = version > 1
+
+      [period, field, cast, needs_comment, declarative, version]
+    end
+
+    protected
+
+    def execute(*args)
+      PgSlice::CLI.instance.send(:execute, *args)
+    end
+
+    def quote_ident(value)
+      PG::Connection.quote_ident(value)
+    end
+
+    def regclass
+      "'#{quote_table}'::regclass"
+    end
+
+    def sql_date(time, cast, add_cast = true)
+      if cast == "timestamptz"
+        fmt = "%Y-%m-%d %H:%M:%S UTC"
+      else
+        fmt = "%Y-%m-%d"
+      end
+      str = "'#{time.strftime(fmt)}'"
+      add_cast ? "#{str}::#{cast}" : str
+    end
+  end
+end

data/lib/pgslice/version.rb

@@ -1,3 +1,3 @@
 module PgSlice
-  VERSION = "0.4.2"
+  VERSION = "0.4.7"
 end

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: pgslice
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.7
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-24 00:00:00.000000000 Z
+date: 2020-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: slop
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.2
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -92,6 +92,16 @@ files:
 - README.md
 - exe/pgslice
 - lib/pgslice.rb
+- lib/pgslice/cli.rb
+- lib/pgslice/cli/add_partitions.rb
+- lib/pgslice/cli/analyze.rb
+- lib/pgslice/cli/fill.rb
+- lib/pgslice/cli/prep.rb
+- lib/pgslice/cli/swap.rb
+- lib/pgslice/cli/unprep.rb
+- lib/pgslice/cli/unswap.rb
+- lib/pgslice/helpers.rb
+- lib/pgslice/table.rb
 - lib/pgslice/version.rb
 homepage: https://github.com/ankane/pgslice
 licenses:
@@ -112,8 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Postgres partitioning as easy as pie