cipherstash-pg 1.0.0.beta.4-x86_64-darwin

data/sample/losample.rb

@@ -0,0 +1,51 @@
+require("cipherstash-pg")
+SAMPLE_WRITE_DATA = "some sample data"
+SAMPLE_EXPORT_NAME = "lowrite.txt"
+conn = CipherStashPG.connect(:dbname => "test", :host => "localhost", :port => 5432)
+puts(((((("dbname: " + conn.db) + "\thost: ") + conn.host) + "\tuser: ") + conn.user))
+puts("Beginning transaction")
+conn.exec("BEGIN")
+puts("Import test:")
+puts(("  importing %s" % ["(string)"]))
+oid = conn.lo_import("(string)")
+puts(("  imported as large object %d" % [oid]))
+puts("Read test:")
+fd = conn.lo_open(oid, (CipherStashPG::INV_READ | CipherStashPG::INV_WRITE))
+conn.lo_lseek(fd, 0, CipherStashPG::SEEK_SET)
+buf = conn.lo_read(fd, 50)
+puts(("  read: %p" % [buf]))
+if buf =~ /require 'pg'/ then
+  puts("  read was ok!")
+end
+puts("Write test:")
+conn.lo_lseek(fd, 0, CipherStashPG::SEEK_END)
+buf = SAMPLE_WRITE_DATA.dup
+totalbytes = 0
+until buf.empty? do
+  (bytes = conn.lo_write(fd, buf)
+  buf.slice!(0, bytes)
+  totalbytes = (totalbytes + bytes))
+end
+puts(("  appended %d bytes" % [totalbytes]))
+puts("Export test:")
+File.unlink(SAMPLE_EXPORT_NAME) if File.exist?(SAMPLE_EXPORT_NAME)
+conn.lo_export(oid, SAMPLE_EXPORT_NAME)
+puts("  success!") if File.exist?(SAMPLE_EXPORT_NAME)
+puts(("  exported as %s (%d bytes)" % [SAMPLE_EXPORT_NAME, File.size(SAMPLE_EXPORT_NAME)]))
+conn.exec("COMMIT")
+puts("End of transaction.")
+puts("Testing read and delete from a new transaction:")
+puts("  starting a new transaction")
+conn.exec("BEGIN")
+fd = conn.lo_open(oid, CipherStashPG::INV_READ)
+puts("  reopened okay.")
+conn.lo_lseek(fd, 50, CipherStashPG::SEEK_END)
+buf = conn.lo_read(fd, 50)
+puts("  read okay.") if (buf == SAMPLE_WRITE_DATA)
+puts("Closing and unlinking:")
+conn.lo_close(fd)
+puts("  closed.")
+conn.lo_unlink(oid)
+puts("  unlinked.")
+conn.exec("COMMIT")
+puts("Done.")

data/sample/minimal-testcase.rb

@@ -0,0 +1,6 @@
+require("cipherstash-pg")
+conn = CipherStashPG.connect(:dbname => "test")
+$stderr.puts("---", RUBY_DESCRIPTION, CipherStashPG.version_string(true), "Server version: #{conn.server_version}", "Client version: #{CipherStashPG.library_version}", "---")
+result = conn.exec("SELECT * from pg_stat_activity")
+$stderr.puts("Expected this to return: [\"select * from pg_stat_activity\"]")
+p(result.field_values("current_query"))

data/sample/notify_wait.rb

@@ -0,0 +1,26 @@
+BEGIN {
+  require("pathname")
+  basedir = Pathname.new("(string)").expand_path.dirname.parent
+  libdir = (basedir + "lib")
+  $LOAD_PATH.unshift(libdir.to_s) unless $LOAD_PATH.include?(libdir.to_s)
+}
+require("cipherstash-pg")
+TRIGGER_TABLE = "\n\tCREATE TABLE IF NOT EXISTS test ( message text );\n"
+TRIGGER_FUNCTION = "\nCREATE OR REPLACE FUNCTION notify_test()\nRETURNS TRIGGER\nLANGUAGE plpgsql\nAS $$\n    BEGIN\n        NOTIFY woo;\n        RETURN NULL;\n    END\n$$\n"
+DROP_TRIGGER = "\nDROP TRIGGER IF EXISTS notify_trigger ON test\n"
+TRIGGER = "\nCREATE TRIGGER notify_trigger\nAFTER UPDATE OR INSERT OR DELETE\nON test\nFOR EACH STATEMENT\nEXECUTE PROCEDURE notify_test();\n"
+conn = CipherStashPG.connect(:dbname => "test")
+conn.exec(TRIGGER_TABLE)
+conn.exec(TRIGGER_FUNCTION)
+conn.exec(DROP_TRIGGER)
+conn.exec(TRIGGER)
+conn.exec("LISTEN woo")
+notifications = []
+puts("Now switch to a different term and run:", "", "  psql test -c \"insert into test values ('A message.')\"", "")
+puts("Waiting up to 30 seconds for for an event!")
+conn.wait_for_notify(30) { |notify, pid| (notifications << [pid, notify]) }
+if notifications.empty? then
+  puts("Awww, I didn't see any events.")
+else
+  puts(("I got one from pid %d: %s" % notifications.first))
+end

data/sample/pg_statistics.rb

@@ -0,0 +1,104 @@
+require("ostruct")
+require("optparse")
+require("etc")
+require("cipherstash-pg")
+class Stats
+  VERSION = "Id"
+  
+  def initialize(opts)
+    @opts = opts
+    @db = CipherStashPG.connect(:dbname => opts.database, :host => opts.host, :port => opts.port, :user => opts.user, :password => opts.pass, :sslmode => "prefer")
+    @last = nil
+  end
+  
+  public
+  
+  def run
+    run_count = 0
+    loop do
+      current_stat = self.get_stats
+      if @last.nil? then
+        @last = current_stat
+        sleep(@opts.interval)
+        next
+      end
+      if ((run_count == 0) or ((run_count % 50) == 0)) then
+        puts(("%-20s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s" % ["time", "commits", "rollbks", "blksrd", "blkshit", "bkends", "seqscan", "seqtprd", "idxscn", "idxtrd", "ins", "upd", "del", "locks", "activeq"]))
+      end
+      delta = current_stat.inject({}) do |h, pair|
+        stat, val = *pair
+        if ["activeq", "locks", "bkends"].include?(stat) then
+          h[stat] = current_stat[stat].to_i
+        else
+          h[stat] = (current_stat[stat].to_i - @last[stat].to_i)
+        end
+        h
+      end
+      delta["time"] = Time.now.strftime("%F %T")
+      puts(("%-20s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s%12s" % [delta["time"], delta["commits"], delta["rollbks"], delta["blksrd"], delta["blkshit"], delta["bkends"], delta["seqscan"], delta["seqtprd"], delta["idxscn"], delta["idxtrd"], delta["ins"], delta["upd"], delta["del"], delta["locks"], delta["activeq"]]))
+      @last = current_stat
+      run_count = (run_count + 1)
+      sleep(@opts.interval)
+    end
+  end
+  
+  def get_stats
+    res = @db.exec(("\n\t\t\tSELECT\n\t\t\t\tMAX(stat_db.xact_commit)       AS commits,\n\t\t\t\tMAX(stat_db.xact_rollback)     AS rollbks,\n\t\t\t\tMAX(stat_db.blks_read)         AS blksrd,\n\t\t\t\tMAX(stat_db.blks_hit)          AS blkshit,\n\t\t\t\tMAX(stat_db.numbackends)       AS bkends,\n\t\t\t\tSUM(stat_tables.seq_scan)      AS seqscan,\n\t\t\t\tSUM(stat_tables.seq_tup_read)  AS seqtprd,\n\t\t\t\tSUM(stat_tables.idx_scan)      AS idxscn,\n\t\t\t\tSUM(stat_tables.idx_tup_fetch) AS idxtrd,\n\t\t\t\tSUM(stat_tables.n_tup_ins)     AS ins,\n\t\t\t\tSUM(stat_tables.n_tup_upd)     AS upd,\n\t\t\t\tSUM(stat_tables.n_tup_del)     AS del,\n\t\t\t\tMAX(stat_locks.locks)          AS locks,\n\t\t\t\tMAX(activity.sess)             AS activeq\n\t\t\tFROM\n\t\t\t\tpg_stat_database    AS stat_db,\n\t\t\t\tpg_stat_user_tables AS stat_tables,\n\t\t\t\t(SELECT COUNT(*) AS locks FROM pg_locks ) AS stat_locks,\n\t\t\t\t(SELECT COUNT(*) AS sess FROM pg_stat_activity WHERE current_query <> '<IDLE>') AS activity\n\t\t\tWHERE\n\t\t\t\tstat_db.datname = '%s';\n\t\t" % [@opts.database]))
+    return res[0]
+  end
+end
+def parse_args(args)
+  options = OpenStruct.new
+  options.database = Etc.getpwuid(Process.uid).name
+  options.host = "127.0.0.1"
+  options.port = 5432
+  options.user = Etc.getpwuid(Process.uid).name
+  options.sslmode = "disable"
+  options.interval = 5
+  opts = OptionParser.new do |opts|
+    opts.banner = "Usage: #{$0} [options]"
+    opts.separator("")
+    opts.separator("Connection options:")
+    opts.on("-d", "--database DBNAME", "specify the database to connect to (default: \"#{options.database}\")") do |db|
+      options.database = db
+    end
+    opts.on("-h", "--host HOSTNAME", "database server host") do |host|
+      options.host = host
+    end
+    opts.on("-p", "--port PORT", Integer, "database server port (default: \"#{options.port}\")") do |port|
+      options.port = port
+    end
+    opts.on("-U", "--user NAME", "database user name (default: \"#{options.user}\")") do |user|
+      options.user = user
+    end
+    opts.on("-W", "force password prompt") do |pw|
+      print("Password: ")
+      begin
+        (system("stty -echo")
+        options.pass = gets.chomp)
+      ensure
+        (system("stty echo")
+        puts)
+      end
+    end
+    opts.separator("")
+    opts.separator("Other options:")
+    opts.on("-i", "--interval SECONDS", Integer, "refresh interval in seconds (default: \"#{options.interval}\")") do |seconds|
+      options.interval = seconds
+    end
+    opts.on_tail("--help", "show this help, then exit") do
+      $stderr.puts(opts)
+      exit
+    end
+    opts.on_tail("--version", "output version information, then exit") do
+      puts(Stats::VERSION)
+      exit
+    end
+  end
+  opts.parse!(args)
+  return options
+end
+if ("(string)" == $0) then
+  $stdout.sync = true
+  Stats.new(parse_args(ARGV)).run
+end

data/sample/replication_monitor.rb

@@ -0,0 +1,123 @@
+require("ostruct")
+require("optparse")
+require("pathname")
+require("etc")
+require("cipherstash-pg")
+require("pp")
+class PGMonitor
+  VERSION = "Id"
+  
+  LAG_ALERT = 32
+  
+  def initialize(opts, hosts)
+    @opts = opts
+    @master = hosts.shift
+    @slaves = hosts
+    @current_wal = {}
+    @failures = []
+  end
+  
+  attr_reader(:opts, :current_wal, :master, :slaves, :failures)
+  
+  def check
+    @failures = []
+    return unless self.get_current_wal
+    self.slaves.each do |slave|
+      begin
+        (slave_db = CipherStashPG.connect(:dbname => self.opts.database, :host => slave, :port => self.opts.port, :user => self.opts.user, :password => self.opts.pass, :sslmode => "prefer")
+        xlog = slave_db.exec("SELECT pg_last_xlog_receive_location()").getvalue(0, 0)
+        slave_db.close
+        lag_in_megs = ((self.find_lag(xlog).to_f / 1024) / 1024).abs
+        if (lag_in_megs >= LAG_ALERT) then
+          (failures << { :host => slave, :error => ("%0.2fMB behind the master." % [lag_in_megs]) })
+        end)
+      rescue => err
+        (failures << { :host => slave, :error => err.message })
+      end
+    end
+  end
+  
+  protected
+  
+  def get_current_wal
+    (master_db = CipherStashPG.connect(:dbname => self.opts.database, :host => self.master, :port => self.opts.port, :user => self.opts.user, :password => self.opts.pass, :sslmode => "prefer")
+    self.current_wal[:segbytes] = (master_db.exec("SHOW wal_segment_size").getvalue(0, 0).sub(/\D+/, "").to_i << 20)
+    current = master_db.exec("SELECT pg_current_xlog_location()").getvalue(0, 0)
+    self.current_wal[:segment], self.current_wal[:offset] = current.split(/\//)
+    master_db.close
+    return true)
+  rescue => err
+    (self.failures << { :host => self.master, :error => ("Unable to retrieve required info from the master (%s)" % [err.message]) })
+    return false
+  end
+  
+  def find_lag(xlog)
+    s_segment, s_offset = xlog.split(/\//)
+    m_segment = self.current_wal[:segment]
+    m_offset = self.current_wal[:offset]
+    m_segbytes = self.current_wal[:segbytes]
+    return (((m_segment.hex - s_segment.hex) * m_segbytes) + (m_offset.hex - s_offset.hex))
+  end
+end
+def parse_args(args)
+  options = OpenStruct.new
+  options.database = "postgres"
+  options.port = 5432
+  options.user = Etc.getpwuid(Process.uid).name
+  options.sslmode = "prefer"
+  opts = OptionParser.new do |opts|
+    opts.banner = "Usage: #{$0} [options] <master> <slave> [slave2, slave3...]"
+    opts.separator("")
+    opts.separator("Connection options:")
+    opts.on("-d", "--database DBNAME", "specify the database to connect to (default: \"#{options.database}\")") do |db|
+      options.database = db
+    end
+    opts.on("-h", "--host HOSTNAME", "database server host") do |host|
+      options.host = host
+    end
+    opts.on("-p", "--port PORT", Integer, "database server port (default: \"#{options.port}\")") do |port|
+      options.port = port
+    end
+    opts.on("-U", "--user NAME", "database user name (default: \"#{options.user}\")") do |user|
+      options.user = user
+    end
+    opts.on("-W", "force password prompt") do |pw|
+      print("Password: ")
+      begin
+        (system("stty -echo")
+        options.pass = $stdin.gets.chomp)
+      ensure
+        (system("stty echo")
+        puts)
+      end
+    end
+    opts.separator("")
+    opts.separator("Other options:")
+    opts.on_tail("--help", "show this help, then exit") do
+      $stderr.puts(opts)
+      exit
+    end
+    opts.on_tail("--version", "output version information, then exit") do
+      puts(PGMonitor::VERSION)
+      exit
+    end
+  end
+  opts.parse!(args)
+  return options
+end
+if ("(string)" == $0) then
+  opts = parse_args(ARGV)
+  if (ARGV.length < 2) then
+    raise(ArgumentError, "At least two PostgreSQL servers are required.")
+  end
+  mon = PGMonitor.new(opts, ARGV)
+  mon.check
+  if mon.failures.empty? then
+    puts("All is well!")
+    exit(0)
+  else
+    puts("Database replication delayed or broken.")
+    mon.failures.each { |bad| puts(("%s: %s" % [bad[:host], bad[:error]])) }
+    exit(1)
+  end
+end

data/sample/test_binary_values.rb

@@ -0,0 +1,17 @@
+require("cipherstash-pg")
+db = CipherStashPG.connect(:dbname => "test")
+db.exec("DROP TABLE IF EXISTS test")
+db.exec("CREATE TABLE test (a INTEGER, b BYTEA)")
+a = 42
+b = [1, 2, 3]
+db.exec("INSERT INTO test(a, b) VALUES($1::int, $2::bytea)", [a, { :value => b.pack("N*"), :format => 1 }])
+db.exec("SELECT a::int, b::bytea FROM test LIMIT 1", [], 1) do |res|
+  res.nfields.times do |i|
+    puts(("Field %d is: %s, a %s (%s) column from table %p" % [i, res.fname(i), db.exec("SELECT format_type($1,$2)", [res.ftype(i), res.fmod(1)]).getvalue(0, 0), res.fformat(i).zero? ? ("string") : ("binary"), res.ftable(i)]))
+  end
+  res.each do |row|
+    puts("a = #{row["a"].inspect}")
+    puts("a (unpacked) = #{row["a"].unpack("N*").inspect}")
+    puts("b = #{row["b"].unpack("N*").inspect}")
+  end
+end

data/sample/wal_shipper.rb

@@ -0,0 +1,202 @@
+require("pathname")
+require("yaml")
+require("fileutils")
+require("ostruct")
+module WalShipper
+  def log(msg)
+    return unless @debug
+    puts(("WAL Shipper: %s" % [msg]))
+  end
+  
+  class Destination < OpenStruct
+    include(WalShipper)
+    
+    def initialize(dest, debug = false)
+      @debug = debug
+      super(dest)
+      self.validate
+    end
+    
+    protected
+    
+    def validate
+      ["label", "kind"].each do |key|
+        if self.send(key.to_sym).nil? then
+          self.log(("Destination %p missing required '%s' key." % [self, key]))
+          self.invalid = true
+        end
+      end
+      self.path = Pathname.new(self.path) if (self.kind == "file")
+      if (self.kind == "rsync-ssh") then
+        self.port ||= 22
+        self.user = self.user ? ("#{self.user}@") : ("")
+      end
+    end
+  end
+  
+  class Dispatcher
+    include(WalShipper)
+    
+    def initialize(wal, conf)
+      conf.each_pair { |key, val| self.instance_variable_set("@#{key}", val) }
+      @spool = Pathname.new(@spool)
+      (@spool.exist? or raise(("The configured spool directory (%s) doesn't exist." % [@spool])))
+      unless @enabled then
+        self.log(("WAL shipping is disabled, queuing segment %s" % [wal.basename]))
+        exit(1)
+      end
+      @destinations.collect! { |dest| WalShipper::Destination.new(dest, @debug) }.reject do |dest|
+        dest.invalid
+      end.collect do |dest|
+        dest.spool = (@spool + dest.label)
+        dest.spool.mkdir(457) unless dest.spool.exist?
+        dest
+      end
+      @waldir = (@spool + "wal_segments")
+      @waldir.mkdir(457) unless @waldir.exist?
+      self.log(("Copying %s to %s" % [wal.basename, @waldir]))
+      FileUtils.cp(wal, @waldir)
+      @wal = (@waldir + wal.basename)
+    end
+    
+    def link
+      @destinations.each do |dest|
+        self.log(("Linking %s into %s" % [@wal.basename, dest.spool.basename]))
+        FileUtils.ln(@wal, dest.spool, :force => true)
+      end
+    end
+    
+    def dispatch
+      unless @async then
+        self.log("Performing a synchronous dispatch.")
+        @destinations.each { |dest| self.dispatch_dest(dest) }
+        return
+      end
+      tg = ThreadGroup.new
+      if (@async_max.nil? or @async_max.to_i.zero?) then
+        self.log("Performing an asynchronous dispatch: one thread per destination.")
+        @destinations.each do |dest|
+          t = Thread.new do
+            Thread.current.abort_on_exception = true
+            self.dispatch_dest(dest)
+          end
+          tg.add(t)
+        end
+        tg.list.each { |t| t.join }
+        return
+      end
+      self.log(("Performing an asynchronous dispatch: one thread per destination, %d at a time." % [@async_max]))
+      all_dests = @destinations.dup
+      dest_chunks = []
+      until all_dests.empty? do
+        (dest_chunks << all_dests.slice!(0, @async_max))
+      end
+      dest_chunks.each do |chunk|
+        chunk.each do |dest|
+          t = Thread.new do
+            Thread.current.abort_on_exception = true
+            self.dispatch_dest(dest)
+          end
+          tg.add(t)
+        end
+        tg.list.each { |t| t.join }
+      end
+      return
+    end
+    
+    def clean_spool
+      total = 0
+      @waldir.children.each do |wal|
+        total = (total + wal.unlink) if (wal.stat.nlink == 1)
+      end
+      self.log(("Removed %d WAL segment%s." % [total, (total == 1) ? ("") : ("s")]))
+    end
+    
+    protected
+    
+    def ship_rsync_ssh(dest)
+      if dest.host.nil? then
+        self.log(("Destination %p missing required 'host' key.  WAL is queued." % [dest.host]))
+        return
+      end
+      rsync_flags = "-zc"
+      ssh_string = ("%s -o ConnectTimeout=%d -o StrictHostKeyChecking=no -p %d" % [@ssh, (@ssh_timeout or 10), dest.port])
+      src_string = ""
+      dst_string = ("%s%s:%s/" % [dest.user, dest.host, dest.path])
+      if (dest.spool.children.length > 1) then
+        src_string = (dest.spool.to_s + "/")
+        (rsync_flags << "r")
+      else
+        src_string = (dest.spool + @wal.basename)
+      end
+      ship_wal_cmd = [@rsync, @debug ? ((rsync_flags << "vh")) : ((rsync_flags << "q")), "--remove-source-files", "-e", ssh_string, src_string, dst_string]
+      self.log(("Running command '%s'" % [ship_wal_cmd.join(" ")]))
+      system(*ship_wal_cmd)
+      unless $?.success? then
+        self.log(("Ack!  Error while shipping to %p, WAL is queued." % [dest.label]))
+        system(@error_cmd, dest.label) if @error_cmd
+      end
+    end
+    
+    def ship_file(dest)
+      if dest.path.nil? then
+        self.log(("Destination %p missing required 'path' key.  WAL is queued." % [dest]))
+        return
+      end
+      dest.path.mkdir(457) unless dest.path.exist?
+      if (dest.spool.children.length > 1) then
+        dest.spool.children.each do |wal|
+          wal.unlink if self.copy_file(wal, dest.path, dest.label, dest.compress)
+        end
+      else
+        wal = (dest.spool + @wal.basename)
+        wal.unlink if self.copy_file(wal, dest.path, dest.label, dest.compress)
+      end
+    end
+    
+    def copy_file(wal, path, label, compress = false)
+      (dest_file = (path + wal.basename)
+      FileUtils.cp(wal, dest_file)
+      if compress then
+        system(*["gzip", "-f", dest_file])
+        raise(("Error while compressing: %s" % [wal.basename])) unless $?.success?
+      end
+      self.log(("Copied %s%s to %s." % [wal.basename, compress ? (" (and compressed)") : (""), path]))
+      return true)
+    rescue => err
+      self.log(("Ack!  Error while copying '%s' (%s) to %p, WAL is queued." % [wal.basename, err.message, path]))
+      system(@error_cmd, label) if @error_cmd
+      return false
+    end
+    
+    def dispatch_dest(dest)
+      if (not dest.enabled.nil?) and (not dest.enabled) then
+        self.log(("Skipping explicitly disabled destination %p, WAL is queued." % [dest.label]))
+        return
+      end
+      meth = ("ship_" + dest.kind.gsub(/-/, "_")).to_sym
+      if WalShipper::Dispatcher.method_defined?(meth) then
+        self.send(meth, dest)
+      else
+        self.log(("Unknown destination kind %p for %p.  WAL is queued." % [dest.kind, dest.label]))
+      end
+    end
+  end
+end
+if ("(string)" == $0) then
+  CONFIG_DIR = (Pathname.new("(string)").dirname.parent + "etc")
+  CONFIG = (CONFIG_DIR + "wal_shipper.conf")
+  unless CONFIG.exist? then
+    CONFIG_DIR.mkdir(457) unless CONFIG_DIR.exist?
+    CONFIG.open("w") { |conf| conf.print(DATA.read) }
+    CONFIG.chmod(420)
+    puts("No WAL shipping configuration found, default file created.")
+  end
+  (wal = ARGV[0] or raise("No WAL file was specified on the command line."))
+  wal = Pathname.new(wal)
+  conf = YAML.load(CONFIG.read)
+  shipper = WalShipper::Dispatcher.new(wal, conf)
+  shipper.link
+  shipper.dispatch
+  shipper.clean_spool
+end