pg 0.13.2 → 0.14.0.pre.351

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,94 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'pg'
4
+
5
+ # This is another example of how to use COPY FROM, this time as a
6
+ # minimal test case used to try to figure out what was going on in
7
+ # an issue submitted from a user:
8
+ #
9
+ # https://bitbucket.org/ged/ruby-pg/issue/119
10
+ #
11
+
12
+ conn = PG.connect( dbname: 'test' )
13
+ table_name = 'issue_119'
14
+ field_list = %w[name body_weight brain_weight]
15
+ method = 0
16
+ options = { truncate: true }
17
+ sql_parameters = ''
18
+
19
+ conn.set_error_verbosity( PG::PQERRORS_VERBOSE )
20
+ conn.exec( "DROP TABLE IF EXISTS #{table_name}" )
21
+ conn.exec( "CREATE TABLE #{table_name} ( id SERIAL, name TEXT, body_weight REAL, brain_weight REAL )" )
22
+
23
+ text = <<-END_DATA
24
+ Mountain beaver 1.35 465
25
+ Cow 465 423
26
+ Grey wolf 36.33 119.5
27
+ Goat 27.66 115
28
+ Guinea pig 1.04 5.5
29
+ Dipliodocus 11700 50
30
+ Asian elephant 2547 4603
31
+ Donkey 187.1 419
32
+ Horse 521 655
33
+ Potar monkey 10 115
34
+ Cat 3.3 25.6
35
+ Giraffe 529 680
36
+ Gorilla 207 406
37
+ Human 62 1320
38
+ African elephant 6654 5712
39
+ Triceratops 9400 70
40
+ Rhesus monkey 6.8 179
41
+ Kangaroo 35 56
42
+ Golden hamster 0.12 1
43
+ Mouse 0.023 0.4
44
+ Rabbit 2.5 12.1
45
+ Sheep 55.5 175
46
+ Jaguar 100 157
47
+ Chimpanzee 52.16 440
48
+ Brachiosaurus 87000 154.5
49
+ Mole 0.122 3
50
+ Pig 192 18
51
+ END_DATA
52
+
53
+ #ActiveRecord::Base.connection_pool.with_connection do |conn|
54
+ conn.transaction do
55
+ rc = conn #.raw_connection
56
+ rc.exec "TRUNCATE TABLE #{table_name};" if options[:truncate]
57
+ sql = "COPY #{table_name} (#{field_list.join(',')}) FROM STDIN #{sql_parameters} "
58
+ p sql
59
+ rc.exec(sql)
60
+ errmsg = nil # scope this outside of the rescue below so it's visible later
61
+ begin
62
+ if method == 1
63
+ rc.put_copy_data text + "\\.\n"
64
+ else
65
+ text.each_line { |line| rc.put_copy_data(line) }
66
+ end
67
+ rescue Errno => err
68
+ errmsg = "%s while reading copy data: %s" % [err.class.name, err.message]
69
+ puts "an error occured"
70
+ end
71
+
72
+ if errmsg
73
+ rc.put_copy_end(errmsg)
74
+ puts "ERROR #{errmsg}"
75
+ else
76
+ rc.put_copy_end
77
+ end
78
+
79
+ while res = rc.get_result
80
+ st = res.res_status( res.result_status )
81
+ puts "Result of COPY is: %s" % [ st ]
82
+ if res.result_status != PG::PGRES_COPY_IN
83
+ puts res.error_message
84
+ end
85
+ end
86
+ puts "end"
87
+ end #transaction
88
+ #end #connection
89
+
90
+ conn.exec( "SELECT name, brain_weight FROM #{table_name}" ) do |res|
91
+ p res.values
92
+ end
93
+
94
+
@@ -0,0 +1,17 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'pg'
4
+
5
+ conn = PG.connect( :dbname => 'test' )
6
+ $stderr.puts '---',
7
+ RUBY_DESCRIPTION,
8
+ PG.version_string( true ),
9
+ "Server version: #{conn.server_version}",
10
+ "Client version: #{PG.respond_to?( :library_version ) ? PG.library_version : 'unknown'}",
11
+ '---'
12
+
13
+ result = conn.exec( "SELECT * from pg_stat_activity" )
14
+
15
+ $stderr.puts %Q{Expected this to return: ["select * from pg_stat_activity"]}
16
+ p result.field_values( 'current_query' )
17
+
@@ -0,0 +1,294 @@
1
+ #!/usr/bin/env ruby
2
+ # vim: set noet nosta sw=4 ts=4 :
3
+ #
4
+ # PostgreSQL statistic gatherer.
5
+ # Mahlon E. Smith <mahlon@martini.nu>
6
+ #
7
+ # Based on queries by Kenny Gorman.
8
+ # http://www.kennygorman.com/wordpress/?page_id=491
9
+ #
10
+ # An example gnuplot input script is included in the __END__ block
11
+ # of this script. Using it, you can feed the output this script
12
+ # generates to gnuplot (after removing header lines) to generate
13
+ # some nice performance charts.
14
+ #
15
+
16
+ begin
17
+ require 'ostruct'
18
+ require 'optparse'
19
+ require 'etc'
20
+ require 'pg'
21
+
22
+ rescue LoadError # 1.8 support
23
+ unless Object.const_defined?( :Gem )
24
+ require 'rubygems'
25
+ retry
26
+ end
27
+ raise
28
+ end
29
+
30
+
31
+ ### PostgreSQL Stats. Fetch information from pg_stat_* tables.
32
+ ### Optionally run in a continuous loop, displaying deltas.
33
+ ###
34
+ class Stats
35
+ VERSION = %q$Id: pg_statistics.rb,v 36ca5b412583 2012/04/17 23:32:25 mahlon $
36
+
37
+ def initialize( opts )
38
+ @opts = opts
39
+ @db = PG.connect(
40
+ :dbname => opts.database,
41
+ :host => opts.host,
42
+ :port => opts.port,
43
+ :user => opts.user,
44
+ :password => opts.pass,
45
+ :sslmode => 'prefer'
46
+ )
47
+ @last = nil
48
+ end
49
+
50
+ ######
51
+ public
52
+ ######
53
+
54
+ ### Primary loop. Gather statistics and generate deltas.
55
+ ###
56
+ def run
57
+ run_count = 0
58
+
59
+ loop do
60
+ current_stat = self.get_stats
61
+
62
+ # First run, store and continue
63
+ #
64
+ if @last.nil?
65
+ @last = current_stat
66
+ sleep @opts.interval
67
+ next
68
+ end
69
+
70
+ # headers
71
+ #
72
+ if run_count == 0 || run_count % 50 == 0
73
+ puts "%-20s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s" % %w[
74
+ time commits rollbks blksrd blkshit bkends seqscan
75
+ seqtprd idxscn idxtrd ins upd del locks activeq
76
+ ]
77
+ end
78
+
79
+ # calculate deltas
80
+ #
81
+ delta = current_stat.inject({}) do |h, pair|
82
+ stat, val = *pair
83
+
84
+ if %w[ activeq locks bkends ].include?( stat )
85
+ h[stat] = current_stat[stat].to_i
86
+ else
87
+ h[stat] = current_stat[stat].to_i - @last[stat].to_i
88
+ end
89
+
90
+ h
91
+ end
92
+ delta[ 'time' ] = Time.now.strftime('%F %T')
93
+
94
+ # new values
95
+ #
96
+ puts "%-20s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s" % [
97
+ delta['time'], delta['commits'], delta['rollbks'], delta['blksrd'],
98
+ delta['blkshit'], delta['bkends'], delta['seqscan'],
99
+ delta['seqtprd'], delta['idxscn'], delta['idxtrd'],
100
+ delta['ins'], delta['upd'], delta['del'], delta['locks'], delta['activeq']
101
+ ]
102
+
103
+ @last = current_stat
104
+ run_count += 1
105
+ sleep @opts.interval
106
+ end
107
+ end
108
+
109
+
110
+ ### Query the database for performance measurements. Returns a hash.
111
+ ###
112
+ def get_stats
113
+ res = @db.exec %Q{
114
+ SELECT
115
+ MAX(stat_db.xact_commit) AS commits,
116
+ MAX(stat_db.xact_rollback) AS rollbks,
117
+ MAX(stat_db.blks_read) AS blksrd,
118
+ MAX(stat_db.blks_hit) AS blkshit,
119
+ MAX(stat_db.numbackends) AS bkends,
120
+ SUM(stat_tables.seq_scan) AS seqscan,
121
+ SUM(stat_tables.seq_tup_read) AS seqtprd,
122
+ SUM(stat_tables.idx_scan) AS idxscn,
123
+ SUM(stat_tables.idx_tup_fetch) AS idxtrd,
124
+ SUM(stat_tables.n_tup_ins) AS ins,
125
+ SUM(stat_tables.n_tup_upd) AS upd,
126
+ SUM(stat_tables.n_tup_del) AS del,
127
+ MAX(stat_locks.locks) AS locks,
128
+ MAX(activity.sess) AS activeq
129
+ FROM
130
+ pg_stat_database AS stat_db,
131
+ pg_stat_user_tables AS stat_tables,
132
+ (SELECT COUNT(*) AS locks FROM pg_locks ) AS stat_locks,
133
+ (SELECT COUNT(*) AS sess FROM pg_stat_activity WHERE current_query <> '<IDLE>') AS activity
134
+ WHERE
135
+ stat_db.datname = '%s';
136
+ } % [ @opts.database ]
137
+
138
+ return res[0]
139
+ end
140
+ end
141
+
142
+
143
+ ### Parse command line arguments. Return a struct of global options.
144
+ ###
145
+ def parse_args( args )
146
+ options = OpenStruct.new
147
+ options.database = Etc.getpwuid( Process.uid ).name
148
+ options.host = '127.0.0.1'
149
+ options.port = 5432
150
+ options.user = Etc.getpwuid( Process.uid ).name
151
+ options.sslmode = 'disable'
152
+ options.interval = 5
153
+
154
+ opts = OptionParser.new do |opts|
155
+ opts.banner = "Usage: #{$0} [options]"
156
+
157
+ opts.separator ''
158
+ opts.separator 'Connection options:'
159
+
160
+ opts.on( '-d', '--database DBNAME',
161
+ "specify the database to connect to (default: \"#{options.database}\")" ) do |db|
162
+ options.database = db
163
+ end
164
+
165
+ opts.on( '-h', '--host HOSTNAME', 'database server host' ) do |host|
166
+ options.host = host
167
+ end
168
+
169
+ opts.on( '-p', '--port PORT', Integer,
170
+ "database server port (default: \"#{options.port}\")" ) do |port|
171
+ options.port = port
172
+ end
173
+
174
+ opts.on( '-U', '--user NAME',
175
+ "database user name (default: \"#{options.user}\")" ) do |user|
176
+ options.user = user
177
+ end
178
+
179
+ opts.on( '-W', 'force password prompt' ) do |pw|
180
+ print 'Password: '
181
+ begin
182
+ system 'stty -echo'
183
+ options.pass = gets.chomp
184
+ ensure
185
+ system 'stty echo'
186
+ puts
187
+ end
188
+ end
189
+
190
+ opts.separator ''
191
+ opts.separator 'Other options:'
192
+
193
+ opts.on( '-i', '--interval SECONDS', Integer,
194
+ "refresh interval in seconds (default: \"#{options.interval}\")") do |seconds|
195
+ options.interval = seconds
196
+ end
197
+
198
+ opts.on_tail( '--help', 'show this help, then exit' ) do
199
+ $stderr.puts opts
200
+ exit
201
+ end
202
+
203
+ opts.on_tail( '--version', 'output version information, then exit' ) do
204
+ puts Stats::VERSION
205
+ exit
206
+ end
207
+ end
208
+
209
+ opts.parse!( args )
210
+ return options
211
+ end
212
+
213
+
214
+ ### Go!
215
+ ###
216
+ if __FILE__ == $0
217
+ $stdout.sync = true
218
+ Stats.new( parse_args( ARGV ) ).run
219
+ end
220
+
221
+
222
+ __END__
223
+ ######################################################################
224
+ ### T E R M I N A L O P T I O N S
225
+ ######################################################################
226
+
227
+ #set terminal png nocrop enhanced font arial 8 size '800x600' x000000 xffffff x444444
228
+ #set output 'graph.png'
229
+
230
+ set terminal pdf linewidth 4 size 11,8
231
+ set output 'graph.pdf'
232
+
233
+ #set terminal aqua
234
+
235
+
236
+ ######################################################################
237
+ ### O P T I O N S F O R A L L G R A P H S
238
+ ######################################################################
239
+
240
+ set multiplot layout 2,1 title "PostgreSQL Statistics\n5 second sample rate (smoothed)"
241
+
242
+ set grid x y
243
+ set key right vertical outside
244
+ set key nobox
245
+ set xdata time
246
+ set timefmt "%Y-%m-%d.%H:%M:%S"
247
+ set format x "%l%p"
248
+ set xtic rotate by -45
249
+ input_file = "database_stats.txt"
250
+
251
+ # edit to taste!
252
+ set xrange ["2012-04-16.00:00:00":"2012-04-17.00:00:00"]
253
+
254
+
255
+ ######################################################################
256
+ ### G R A P H 1
257
+ ######################################################################
258
+
259
+ set title "Database Operations and Connection Totals"
260
+ set yrange [0:200]
261
+
262
+ plot \
263
+ input_file using 1:2 title "Commits" with lines smooth bezier, \
264
+ input_file using 1:3 title "Rollbacks" with lines smooth bezier, \
265
+ input_file using 1:11 title "Inserts" with lines smooth bezier, \
266
+ input_file using 1:12 title "Updates" with lines smooth bezier, \
267
+ input_file using 1:13 title "Deletes" with lines smooth bezier, \
268
+ input_file using 1:6 title "Backends (total)" with lines, \
269
+ input_file using 1:15 title "Active queries (total)" with lines smooth bezier
270
+
271
+
272
+ ######################################################################
273
+ ### G R A P H 2
274
+ ######################################################################
275
+
276
+ set title "Backend Performance"
277
+ set yrange [0:10000]
278
+
279
+ plot \
280
+ input_file using 1:4 title "Block (cache) reads" with lines smooth bezier, \
281
+ input_file using 1:5 title "Block (cache) hits" with lines smooth bezier, \
282
+ input_file using 1:7 title "Sequence scans" with lines smooth bezier, \
283
+ input_file using 1:8 title "Sequence tuple reads" with lines smooth bezier, \
284
+ input_file using 1:9 title "Index scans" with lines smooth bezier, \
285
+ input_file using 1:10 title "Index tuple reads" with lines smooth bezier
286
+
287
+
288
+ ######################################################################
289
+ ### C L E A N U P
290
+ ######################################################################
291
+
292
+ unset multiplot
293
+ reset
294
+
@@ -0,0 +1,231 @@
1
+ #!/usr/bin/env ruby
2
+ # vim: set noet nosta sw=4 ts=4 :
3
+ #
4
+ # Get the current WAL segment and offset from a master postgresql
5
+ # server, and compare slave servers to see how far behind they
6
+ # are in MB. This script should be easily modified for use with
7
+ # Nagios/Mon/Monit/Zabbix/whatever, or wrapping it in a display loop,
8
+ # and is suitable for both WAL shipping or streaming forms of replication.
9
+ #
10
+ # Mahlon E. Smith <mahlon@martini.nu>
11
+ #
12
+ # First argument is the master server, all other arguments are treated
13
+ # as slave machines.
14
+ #
15
+ # db_replication.monitor db-master.example.com ...
16
+ #
17
+
18
+ begin
19
+ require 'ostruct'
20
+ require 'optparse'
21
+ require 'pathname'
22
+ require 'etc'
23
+ require 'pg'
24
+ require 'pp'
25
+
26
+ rescue LoadError # 1.8 support
27
+ unless Object.const_defined?( :Gem )
28
+ require 'rubygems'
29
+ retry
30
+ end
31
+ raise
32
+ end
33
+
34
+
35
+ ### A class to encapsulate the PG handles.
36
+ ###
37
+ class PGMonitor
38
+
39
+ VERSION = %q$Id: replication_monitor.rb,v 36ca5b412583 2012/04/17 23:32:25 mahlon $
40
+
41
+ # When to consider a slave as 'behind', measured in WAL segments.
42
+ # The default WAL segment size is 16, so we'll alert after
43
+ # missing two WAL files worth of data.
44
+ #
45
+ LAG_ALERT = 32
46
+
47
+ ### Create a new PGMonitor object.
48
+ ###
49
+ def initialize( opts, hosts )
50
+ @opts = opts
51
+ @master = hosts.shift
52
+ @slaves = hosts
53
+ @current_wal = {}
54
+ @failures = []
55
+ end
56
+
57
+ attr_reader :opts, :current_wal, :master, :slaves, :failures
58
+
59
+
60
+ ### Perform the connections and check the lag.
61
+ ###
62
+ def check
63
+ # clear prior failures, get current xlog info
64
+ @failures = []
65
+ return unless self.get_current_wal
66
+
67
+ # check all slaves
68
+ self.slaves.each do |slave|
69
+ begin
70
+ slave_db = PG.connect(
71
+ :dbname => self.opts.database,
72
+ :host => slave,
73
+ :port => self.opts.port,
74
+ :user => self.opts.user,
75
+ :password => self.opts.pass,
76
+ :sslmode => 'prefer'
77
+ )
78
+
79
+ xlog = slave_db.exec( 'SELECT pg_last_xlog_receive_location()' ).getvalue( 0, 0 )
80
+ slave_db.close
81
+
82
+ lag_in_megs = ( self.find_lag( xlog ).to_f / 1024 / 1024 ).abs
83
+ if lag_in_megs >= LAG_ALERT
84
+ failures << { :host => slave,
85
+ :error => "%0.2fMB behind the master." % [ lag_in_megs ] }
86
+ end
87
+ rescue => err
88
+ failures << { :host => slave, :error => err.message }
89
+ end
90
+ end
91
+ end
92
+
93
+
94
+ #########
95
+ protected
96
+ #########
97
+
98
+ ### Ask the master for the current xlog information, to compare
99
+ ### to slaves. Returns true on succcess. On failure, populates
100
+ ### the failures array and returns false.
101
+ ###
102
+ def get_current_wal
103
+ master_db = PG.connect(
104
+ :dbname => self.opts.database,
105
+ :host => self.master,
106
+ :port => self.opts.port,
107
+ :user => self.opts.user,
108
+ :password => self.opts.pass,
109
+ :sslmode => 'prefer'
110
+ )
111
+
112
+ self.current_wal[ :segbytes ] = master_db.exec( 'SHOW wal_segment_size' ).
113
+ getvalue( 0, 0 ).sub( /\D+/, '' ).to_i << 20
114
+
115
+ current = master_db.exec( 'SELECT pg_current_xlog_location()' ).getvalue( 0, 0 )
116
+ self.current_wal[ :segment ], self.current_wal[ :offset ] = current.split( /\// )
117
+
118
+ master_db.close
119
+ return true
120
+
121
+ # If we can't get any of the info from the master, then there is no
122
+ # point in a comparison with slaves.
123
+ #
124
+ rescue => err
125
+ self.failures << { :host => self.master,
126
+ :error => 'Unable to retrieve required info from the master (%s)' % [ err.message ] }
127
+
128
+ return false
129
+ end
130
+
131
+
132
+ ### Given an +xlog+ position from a slave server, return
133
+ ### the number of bytes the slave needs to replay before it
134
+ ### is caught up to the master.
135
+ ###
136
+ def find_lag( xlog )
137
+ s_segment, s_offset = xlog.split( /\// )
138
+ m_segment = self.current_wal[ :segment ]
139
+ m_offset = self.current_wal[ :offset ]
140
+ m_segbytes = self.current_wal[ :segbytes ]
141
+
142
+ return (( m_segment.hex - s_segment.hex ) * m_segbytes) + ( m_offset.hex - s_offset.hex )
143
+ end
144
+
145
+ end
146
+
147
+
148
+ ### Parse command line arguments. Return a struct of global options.
149
+ ###
150
+ def parse_args( args )
151
+ options = OpenStruct.new
152
+ options.database = 'postgres'
153
+ options.port = 5432
154
+ options.user = Etc.getpwuid( Process.uid ).name
155
+ options.sslmode = 'prefer'
156
+
157
+ opts = OptionParser.new do |opts|
158
+ opts.banner = "Usage: #{$0} [options] <master> <slave> [slave2, slave3...]"
159
+
160
+ opts.separator ''
161
+ opts.separator 'Connection options:'
162
+
163
+ opts.on( '-d', '--database DBNAME',
164
+ "specify the database to connect to (default: \"#{options.database}\")" ) do |db|
165
+ options.database = db
166
+ end
167
+
168
+ opts.on( '-h', '--host HOSTNAME', 'database server host' ) do |host|
169
+ options.host = host
170
+ end
171
+
172
+ opts.on( '-p', '--port PORT', Integer,
173
+ "database server port (default: \"#{options.port}\")" ) do |port|
174
+ options.port = port
175
+ end
176
+
177
+ opts.on( '-U', '--user NAME',
178
+ "database user name (default: \"#{options.user}\")" ) do |user|
179
+ options.user = user
180
+ end
181
+
182
+ opts.on( '-W', 'force password prompt' ) do |pw|
183
+ print 'Password: '
184
+ begin
185
+ system 'stty -echo'
186
+ options.pass = $stdin.gets.chomp
187
+ ensure
188
+ system 'stty echo'
189
+ puts
190
+ end
191
+ end
192
+
193
+ opts.separator ''
194
+ opts.separator 'Other options:'
195
+
196
+ opts.on_tail( '--help', 'show this help, then exit' ) do
197
+ $stderr.puts opts
198
+ exit
199
+ end
200
+
201
+ opts.on_tail( '--version', 'output version information, then exit' ) do
202
+ puts PGMonitor::VERSION
203
+ exit
204
+ end
205
+ end
206
+
207
+ opts.parse!( args )
208
+ return options
209
+ end
210
+
211
+
212
+
213
+ if __FILE__ == $0
214
+ opts = parse_args( ARGV )
215
+ raise ArgumentError, "At least two PostgreSQL servers are required." if ARGV.length < 2
216
+ mon = PGMonitor.new( opts, ARGV )
217
+
218
+ mon.check
219
+ if mon.failures.empty?
220
+ puts "All is well!"
221
+ exit 0
222
+ else
223
+ puts "Database replication delayed or broken."
224
+ mon.failures.each do |bad|
225
+ puts "%s: %s" % [ bad[ :host ], bad[ :error ] ]
226
+ end
227
+ exit 1
228
+ end
229
+ end
230
+
231
+