pg 1.1.0.pre20180730144600 → 1.1.0.pre20180730171000

data/sample/warehouse_partitions.rb

@@ -0,0 +1,311 @@
+# -*- ruby -*-
+# vim: set nosta noet ts=4 sw=4:
+#
+# Script to automatically move partitioned tables and their indexes
+# to a separate area on disk.
+#
+# Mahlon E. Smith <mahlon@martini.nu>
+#
+# Example use case:
+#
+#   - You've got a heavy insert table, such as syslog data.
+#   - This table has a partitioning trigger (or is manually partitioned)
+#     by date, to separate incoming stuff from archival/report stuff.
+#   - You have a tablespace on cheap or slower disk (maybe even
+#     ZFS compressed, or some such!)
+#
+# The only assumption this script makes is that your tables are dated, and
+# the tablespace they're moving into already exists.
+#
+# A full example, using the syslog idea from above, where each child
+# table is date partitioned by a convention of "syslog_YEAR-WEEKOFYEAR":
+#
+#    syslog             # <--- parent
+#    syslog_2012_06     # <--- inherited
+#    syslog_2012_07     # <--- inherited
+#    syslog_2012_08     # <--- inherited
+#    ...
+#
+# You'd run this script like so:
+#
+#    ./warehouse_partitions.rb -F syslog_%Y_%U
+#
+# Assuming this was week 12 of the year, tables syslog_2012_06 through
+# syslog_2012_11 would start sequentially migrating into the tablespace
+# called 'warehouse'.
+#
+
+
+require 'date'
+require 'ostruct'
+require 'optparse'
+require 'pathname'
+require 'etc'
+require 'pg'
+
+
+### A tablespace migration class.
+###
+class PGWarehouse
+
+	def initialize( opts )
+		@opts = opts
+		@db = PG.connect(
+			:dbname   => opts.database,
+			:host     => opts.host,
+			:port     => opts.port,
+			:user     => opts.user,
+			:password => opts.pass,
+			:sslmode  => 'prefer'
+		)
+		@db.exec "SET search_path TO %s" % [ opts.schema ] if opts.schema
+
+		@relations = self.relations
+	end
+
+	attr_reader :db
+
+	######
+	public
+	######
+
+	### Perform the tablespace moves.
+	###
+	def migrate
+		if @relations.empty?
+			$stderr.puts 'No tables were found for warehousing.'
+			return
+		end
+
+		$stderr.puts "Found %d relation%s to move." % [ relations.length, relations.length == 1 ? '' : 's' ]
+		@relations.sort_by{|_,v| v[:name] }.each do |_, val|
+			$stderr.print "  - Moving table '%s' to '%s'... "  % [
+				val[:name], @opts.tablespace
+			]
+
+			if @opts.dryrun
+				$stderr.puts '(not really)'
+
+			else
+				age = self.timer do
+					db.exec "ALTER TABLE %s SET TABLESPACE %s;" % [
+						val[:name], @opts.tablespace
+					]
+				end
+				puts age
+			end
+
+			val[ :indexes ].each do |idx|
+				$stderr.print "      - Moving index '%s' to '%s'... "  % [
+					idx, @opts.tablespace
+				]
+				if @opts.dryrun
+					$stderr.puts '(not really)'
+
+				else
+					age = self.timer do
+						db.exec "ALTER INDEX %s SET TABLESPACE %s;" % [
+							idx, @opts.tablespace
+						]
+					end
+					puts age
+				end
+			end
+		end
+	end
+
+
+	#########
+	protected
+	#########
+
+	### Get OIDs and current tablespaces for everything under the
+	### specified schema.
+	###
+	def relations
+		return @relations if @relations
+		relations = {}
+
+		query =  %q{
+			SELECT c.oid AS oid,
+				c.relname AS name,
+				c.relkind AS kind,
+				t.spcname AS tspace
+			FROM pg_class AS c
+			LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
+			LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
+			WHERE c.relkind = 'r' }
+		query << "AND n.nspname='#{@opts.schema}'" if @opts.schema
+
+		# Get the relations list, along with each element's current tablespace.
+		#
+		self.db.exec( query ) do |res|
+			res.each do |row|
+				relations[ row['oid'] ] = {
+					:name       => row['name'],
+					:tablespace => row['tspace'],
+					:indexes    => [],
+					:parent     => nil
+				}
+			end
+		end
+
+		# Add table inheritence information.
+		#
+		db.exec 'SELECT inhrelid AS oid, inhparent AS parent FROM pg_inherits' do |res|
+			res.each do |row|
+				relations[ row['oid'] ][ :parent ] = row['parent']
+			end
+		end
+
+		# Remove tables that don't qualify for warehousing.
+		#
+		#   - Tables that are not children of a parent
+		#   - Tables that are already in the warehouse tablespace
+		#   - The currently active child (it's likely being written to!)
+		#   - Any table that can't be parsed into the specified format
+		#
+		relations.reject! do |oid, val|
+			begin
+				val[:parent].nil? ||
+				val[:tablespace] == @opts.tablespace ||
+				val[:name] == Time.now.strftime( @opts.format ) ||
+				! DateTime.strptime( val[:name], @opts.format )
+			rescue ArgumentError
+				true
+			end
+		end
+
+		query = %q{
+			SELECT c.oid AS oid,
+				i.indexname AS name
+			FROM pg_class AS c
+			INNER JOIN pg_indexes AS i
+				ON i.tablename = c.relname }
+		query << "AND i.schemaname='#{@opts.schema}'" if @opts.schema
+
+		# Attach index names to tables.
+		#
+		db.exec( query ) do |res|
+			res.each do |row|
+				relations[ row['oid'] ][ :indexes ] << row['name'] if relations[ row['oid'] ]
+			end
+		end
+
+		return relations
+	end
+
+
+	### Wrap arbitrary commands in a human readable timer.
+	###
+	def timer
+		start = Time.now
+		yield
+		age = Time.now - start
+
+		diff = age
+		secs = diff % 60
+		diff = ( diff - secs ) / 60
+		mins = diff % 60
+		diff = ( diff - mins ) / 60
+		hour = diff % 24
+
+		return "%02d:%02d:%02d" % [ hour, mins, secs ]
+	end
+end
+
+
+### Parse command line arguments.  Return a struct of global options.
+###
+def parse_args( args )
+	options          = OpenStruct.new
+	options.database = Etc.getpwuid( Process.uid ).name
+	options.host     = '127.0.0.1'
+	options.port     = 5432
+	options.user     = Etc.getpwuid( Process.uid ).name
+	options.sslmode  = 'prefer'
+	options.tablespace = 'warehouse'
+
+	opts = OptionParser.new do |opts|
+		opts.banner = "Usage: #{$0} [options]"
+
+		opts.separator ''
+		opts.separator 'Connection options:'
+
+		opts.on( '-d', '--database DBNAME',
+				"specify the database to connect to (default: \"#{options.database}\")" ) do |db|
+			options.database = db
+		end
+
+		opts.on( '-h', '--host HOSTNAME', 'database server host' ) do |host|
+			options.host = host
+		end
+
+		opts.on( '-p', '--port PORT', Integer,
+				"database server port (default: \"#{options.port}\")" ) do |port|
+			options.port = port
+		end
+
+		opts.on( '-n', '--schema SCHEMA', String,
+				"operate on the named schema only (default: none)" ) do |schema|
+			options.schema = schema
+		end
+
+		opts.on( '-T', '--tablespace SPACE', String,
+				"move old tables to this tablespace (default: \"#{options.tablespace}\")" ) do |tb|
+			options.tablespace = tb
+		end
+
+		opts.on( '-F', '--tableformat FORMAT', String,
+				"The naming format (strftime) for the inherited tables (default: none)" ) do |format|
+			options.format = format
+		end
+
+		opts.on( '-U', '--user NAME',
+				"database user name (default: \"#{options.user}\")" ) do |user|
+			options.user = user
+		end
+
+		opts.on( '-W', 'force password prompt' ) do |pw|
+			print 'Password: '
+			begin
+				system 'stty -echo'
+				options.pass = gets.chomp
+			ensure
+				system 'stty echo'
+				puts
+			end
+		end
+
+		opts.separator ''
+		opts.separator 'Other options:'
+
+		opts.on_tail( '--dry-run', "don't actually do anything" ) do
+			options.dryrun = true
+		end
+
+		opts.on_tail( '--help', 'show this help, then exit' ) do
+			$stderr.puts opts
+			exit
+		end
+
+		opts.on_tail( '--version', 'output version information, then exit' ) do
+			puts Stats::VERSION
+			exit
+		end
+	end
+
+	opts.parse!( args )
+	return options
+end
+
+
+if __FILE__ == $0
+	opts = parse_args( ARGV )
+	raise ArgumentError, "A naming format (-F) is required." unless opts.format
+
+	$stdout.sync = true
+	PGWarehouse.new( opts ).migrate
+end
+
+

data/spec/pg/connection_sync_spec.rb

@@ -0,0 +1,41 @@
+# -*- rspec -*-
+#encoding: utf-8
+
+require_relative '../helpers'
+
+context "running with sync_* methods" do
+	before :each do
+		PG::Connection.async_api = false
+	end
+
+	after :each do
+		PG::Connection.async_api = true
+	end
+
+	fname = File.expand_path("../connection_spec.rb", __FILE__)
+	eval File.read(fname, encoding: __ENCODING__), binding, fname
+
+
+	it "enables/disables async/sync methods by #async_api" do
+		[true, false].each do |async|
+			PG::Connection.async_api = async
+
+			start = Time.now
+			t = Thread.new do
+				@conn.exec( 'select pg_sleep(1)' )
+			end
+			sleep 0.1
+
+			t.kill
+			t.join
+			dt = Time.now - start
+
+			if async
+				expect( dt ).to be < 1.0
+			else
+				expect( dt ).to be >= 1.0
+			end
+		end
+	end
+
+end

data/spec/pg/tuple_spec.rb

@@ -0,0 +1,266 @@
+# -*- rspec -*-
+# encoding: utf-8
+
+require_relative '../helpers'
+require 'pg'
+require 'objspace'
+
+describe PG::Tuple do
+	let!(:typemap) { PG::BasicTypeMapForResults.new(@conn) }
+	let!(:result2x2) { @conn.exec( "VALUES(1, 'a'), (2, 'b')" ) }
+	let!(:result2x3cast) { @conn.exec( "SELECT * FROM (VALUES(1, TRUE, '3'), (2, FALSE, '4')) AS m (a, b, b)" ).map_types!(typemap) }
+	let!(:tuple0) { result2x2.tuple(0) }
+	let!(:tuple1) { result2x2.tuple(1) }
+	let!(:tuple2) { result2x3cast.tuple(0) }
+	let!(:tuple3) { str = Marshal.dump(result2x3cast.tuple(1)); Marshal.load(str) }
+	let!(:tuple_empty) { PG::Tuple.new }
+
+	describe "[]" do
+		it "returns nil for invalid keys" do
+			expect( tuple0["x"] ).to be_nil
+			expect( tuple0[0.5] ).to be_nil
+			expect( tuple0[2] ).to be_nil
+			expect( tuple0[-3] ).to be_nil
+			expect{ tuple_empty[0] }.to raise_error(TypeError)
+		end
+
+		it "supports array like access" do
+			expect( tuple0[0] ).to eq( "1" )
+			expect( tuple0[1] ).to eq( "a" )
+			expect( tuple1[0] ).to eq( "2" )
+			expect( tuple1[1] ).to eq( "b" )
+			expect( tuple2[0] ).to eq( 1 )
+			expect( tuple2[1] ).to eq( true )
+			expect( tuple2[2] ).to eq( "3" )
+			expect( tuple3[0] ).to eq( 2 )
+			expect( tuple3[1] ).to eq( false )
+			expect( tuple3[2] ).to eq( "4" )
+		end
+
+		it "supports negative indices" do
+			expect( tuple0[-2] ).to eq( "1" )
+			expect( tuple0[-1] ).to eq( "a" )
+		end
+
+		it "supports hash like access" do
+			expect( tuple0["column1"] ).to eq( "1" )
+			expect( tuple0["column2"] ).to eq( "a" )
+			expect( tuple2["a"] ).to eq( 1 )
+			expect( tuple2["b"] ).to eq( "3" )
+			expect( tuple0["x"] ).to be_nil
+		end
+
+		it "casts lazy and caches result" do
+			a = []
+			deco = Class.new(PG::SimpleDecoder) do
+				define_method(:decode) do |*args|
+					a << args
+					args.last
+				end
+			end.new
+
+			result2x2.map_types!(PG::TypeMapByColumn.new([deco, deco]))
+			t = result2x2.tuple(1)
+
+			# cast and cache at first call to [0]
+			a.clear
+			expect( t[0] ).to eq( 0 )
+			expect( a ).to eq([["2", 1, 0]])
+
+			# use cache at second call to [0]
+			a.clear
+			expect( t[0] ).to eq( 0 )
+			expect( a ).to eq([])
+
+			# cast and cache at first call to [1]
+			a.clear
+			expect( t[1] ).to eq( 1 )
+			expect( a ).to eq([["b", 1, 1]])
+		end
+	end
+
+	describe "fetch" do
+		it "raises proper errors for invalid keys" do
+			expect{ tuple0.fetch("x") }.to raise_error(KeyError)
+			expect{ tuple0.fetch(0.5) }.to raise_error(KeyError)
+			expect{ tuple0.fetch(2) }.to raise_error(IndexError)
+			expect{ tuple0.fetch(-3) }.to raise_error(IndexError)
+			expect{ tuple0.fetch(-3) }.to raise_error(IndexError)
+			expect{ tuple_empty[0] }.to raise_error(TypeError)
+		end
+
+		it "supports array like access" do
+			expect( tuple0.fetch(0) ).to eq( "1" )
+			expect( tuple0.fetch(1) ).to eq( "a" )
+		end
+
+		it "supports default value for indices" do
+			expect( tuple0.fetch(2, 42) ).to eq( 42 )
+			expect( tuple0.fetch(2){43} ).to eq( 43 )
+		end
+
+		it "supports negative indices" do
+			expect( tuple0.fetch(-2) ).to eq( "1" )
+			expect( tuple0.fetch(-1) ).to eq( "a" )
+		end
+
+		it "supports hash like access" do
+			expect( tuple0.fetch("column1") ).to eq( "1" )
+			expect( tuple0.fetch("column2") ).to eq( "a" )
+			expect( tuple2.fetch("a") ).to eq( 1 )
+			expect( tuple2.fetch("b") ).to eq( "3" )
+		end
+
+		it "supports default value for name keys" do
+			expect( tuple0.fetch("x", "defa") ).to eq("defa")
+			expect( tuple0.fetch("x"){"defa"} ).to eq("defa")
+		end
+	end
+
+	describe "each" do
+		it "can be used as an enumerator" do
+			expect( tuple0.each ).to be_kind_of(Enumerator)
+			expect( tuple0.each.to_a ).to eq( [["column1", "1"], ["column2", "a"]] )
+			expect( tuple1.each.to_a ).to eq( [["column1", "2"], ["column2", "b"]] )
+			expect( tuple2.each.to_a ).to eq( [["a", 1], ["b", true], ["b", "3"]] )
+			expect( tuple3.each.to_a ).to eq( [["a", 2], ["b", false], ["b", "4"]] )
+			expect{ tuple_empty.each }.to raise_error(TypeError)
+		end
+
+		it "can be used with block" do
+			a = []
+			tuple0.each do |*v|
+				a << v
+			end
+			expect( a ).to eq( [["column1", "1"], ["column2", "a"]] )
+		end
+	end
+
+	describe "each_value" do
+		it "can be used as an enumerator" do
+			expect( tuple0.each_value ).to be_kind_of(Enumerator)
+			expect( tuple0.each_value.to_a ).to eq( ["1", "a"] )
+			expect( tuple1.each_value.to_a ).to eq( ["2", "b"] )
+			expect( tuple2.each_value.to_a ).to eq( [1, true, "3"] )
+			expect( tuple3.each_value.to_a ).to eq( [2, false, "4"] )
+			expect{ tuple_empty.each_value }.to raise_error(TypeError)
+		end
+
+		it "can be used with block" do
+			a = []
+			tuple0.each_value do |v|
+				a << v
+			end
+			expect( a ).to eq( ["1", "a"] )
+		end
+	end
+
+	it "responds to values" do
+		expect( tuple0.values ).to eq( ["1", "a"] )
+		expect( tuple3.values ).to eq( [2, false, "4"] )
+		expect{ tuple_empty.values }.to raise_error(TypeError)
+	end
+
+	it "responds to key?" do
+		expect( tuple1.key?("column1") ).to eq( true )
+		expect( tuple1.key?("other") ).to eq( false )
+		expect( tuple1.has_key?("column1") ).to eq( true )
+		expect( tuple1.has_key?("other") ).to eq( false )
+	end
+
+	it "responds to keys" do
+		expect( tuple0.keys ).to eq( ["column1", "column2"] )
+		expect( tuple2.keys ).to eq( ["a", "b", "b"] )
+	end
+
+	describe "each_key" do
+		it "can be used as an enumerator" do
+			expect( tuple0.each_key ).to be_kind_of(Enumerator)
+			expect( tuple0.each_key.to_a ).to eq( ["column1", "column2"] )
+			expect( tuple2.each_key.to_a ).to eq( ["a", "b", "b"] )
+		end
+
+		it "can be used with block" do
+			a = []
+			tuple0.each_key do |v|
+				a << v
+			end
+			expect( a ).to eq( ["column1", "column2"] )
+		end
+	end
+
+	it "responds to length" do
+		expect( tuple0.length ).to eq( 2 )
+		expect( tuple0.size ).to eq( 2 )
+		expect( tuple2.size ).to eq( 3 )
+	end
+
+	it "responds to index" do
+		expect( tuple0.index("column1") ).to eq( 0 )
+		expect( tuple0.index("column2") ).to eq( 1 )
+		expect( tuple0.index("x") ).to eq( nil )
+		expect( tuple2.index("a") ).to eq( 0 )
+		expect( tuple2.index("b") ).to eq( 2 )
+	end
+
+	it "can be used as Enumerable" do
+		expect( tuple0.to_a ).to eq( [["column1", "1"], ["column2", "a"]] )
+		expect( tuple1.to_a ).to eq( [["column1", "2"], ["column2", "b"]] )
+		expect( tuple2.to_a ).to eq( [["a", 1], ["b", true], ["b", "3"]] )
+		expect( tuple3.to_a ).to eq( [["a", 2], ["b", false], ["b", "4"]] )
+	end
+
+	it "can be marshaled" do
+		[tuple0, tuple1, tuple2, tuple3].each do |t1|
+			str = Marshal.dump(t1)
+			t2 = Marshal.load(str)
+
+			expect( t2 ).to be_kind_of(t1.class)
+			expect( t2 ).not_to equal(t1)
+			expect( t2.to_a ).to eq(t1.to_a)
+		end
+	end
+
+	it "passes instance variables when marshaled" do
+		t1 = tuple0
+		t1.instance_variable_set("@a", 4711)
+		str = Marshal.dump(t1)
+		t2 = Marshal.load(str)
+
+		expect( t2.instance_variable_get("@a") ).to eq( 4711 )
+	end
+
+	it "can't be marshaled when empty" do
+		expect{ Marshal.dump(tuple_empty) }.to raise_error(TypeError)
+	end
+
+	it "should give account about memory usage" do
+		expect( ObjectSpace.memsize_of(tuple0) ).to be > 40
+		expect( ObjectSpace.memsize_of(tuple_empty) ).to be > 0
+	end
+
+	it "should override #inspect" do
+		expect( tuple1.inspect ).to eq('#<PG::Tuple column1: "2", column2: "b">')
+		expect( tuple2.inspect ).to eq('#<PG::Tuple a: 1, b: true, b: "3">')
+		expect{ tuple_empty.inspect }.to raise_error(TypeError)
+	end
+
+	context "with cleared result" do
+		it "should raise an error when materialized fields are used" do
+			r = result2x2
+			t = r.tuple(0)
+			t[0] # materialize first field only
+			r.clear
+			expect{ t[1] }.to raise_error(PG::Error)
+			expect{ t.fetch(1) }.to raise_error(PG::Error)
+			expect{ t.fetch("column2") }.to raise_error(PG::Error)
+			expect{ t.values }.to raise_error(PG::Error)
+
+			expect( t[0] ).to eq( "1" )
+			expect( t.fetch(0) ).to eq( "1" )
+			expect( t.fetch("column1") ).to eq( "1" )
+
+			expect{ t.values }.to raise_error(PG::Error)
+		end
+	end
+end