cipherstash-pg 1.0.0.beta.1-x86_64-linux

data/sample/warehouse_partitions.rb

@@ -0,0 +1,311 @@
+# -*- ruby -*-
+# vim: set nosta noet ts=4 sw=4:
+#
+# Script to automatically move partitioned tables and their indexes
+# to a separate area on disk.
+#
+# Mahlon E. Smith <mahlon@martini.nu>
+#
+# Example use case:
+#
+#   - You've got a heavy insert table, such as syslog data.
+#   - This table has a partitioning trigger (or is manually partitioned)
+#     by date, to separate incoming stuff from archival/report stuff.
+#   - You have a tablespace on cheap or slower disk (maybe even
+#     ZFS compressed, or some such!)
+#
+# The only assumption this script makes is that your tables are dated, and
+# the tablespace they're moving into already exists.
+#
+# A full example, using the syslog idea from above, where each child
+# table is date partitioned by a convention of "syslog_YEAR-WEEKOFYEAR":
+#
+#    syslog             # <--- parent
+#    syslog_2012_06     # <--- inherited
+#    syslog_2012_07     # <--- inherited
+#    syslog_2012_08     # <--- inherited
+#    ...
+#
+# You'd run this script like so:
+#
+#    ./warehouse_partitions.rb -F syslog_%Y_%U
+#
+# Assuming this was week 12 of the year, tables syslog_2012_06 through
+# syslog_2012_11 would start sequentially migrating into the tablespace
+# called 'warehouse'.
+#
+
+
+require 'date'
+require 'ostruct'
+require 'optparse'
+require 'pathname'
+require 'etc'
+require 'pg'
+
+
+### A tablespace migration class.
+###
+class PGWarehouse
+
+	def initialize( opts )
+		@opts = opts
+		@db = PG.connect(
+			:dbname   => opts.database,
+			:host     => opts.host,
+			:port     => opts.port,
+			:user     => opts.user,
+			:password => opts.pass,
+			:sslmode  => 'prefer'
+		)
+		@db.exec "SET search_path TO %s" % [ opts.schema ] if opts.schema
+
+		@relations = self.relations
+	end
+
+	attr_reader :db
+
+	######
+	public
+	######
+
+	### Perform the tablespace moves.
+	###
+	def migrate
+		if @relations.empty?
+			$stderr.puts 'No tables were found for warehousing.'
+			return
+		end
+
+		$stderr.puts "Found %d relation%s to move." % [ relations.length, relations.length == 1 ? '' : 's' ]
+		@relations.sort_by{|_,v| v[:name] }.each do |_, val|
+			$stderr.print "  - Moving table '%s' to '%s'... "  % [
+				val[:name], @opts.tablespace
+			]
+
+			if @opts.dryrun
+				$stderr.puts '(not really)'
+
+			else
+				age = self.timer do
+					db.exec "ALTER TABLE %s SET TABLESPACE %s;" % [
+						val[:name], @opts.tablespace
+					]
+				end
+				puts age
+			end
+
+			val[ :indexes ].each do |idx|
+				$stderr.print "      - Moving index '%s' to '%s'... "  % [
+					idx, @opts.tablespace
+				]
+				if @opts.dryrun
+					$stderr.puts '(not really)'
+
+				else
+					age = self.timer do
+						db.exec "ALTER INDEX %s SET TABLESPACE %s;" % [
+							idx, @opts.tablespace
+						]
+					end
+					puts age
+				end
+			end
+		end
+	end
+
+
+	#########
+	protected
+	#########
+
+	### Get OIDs and current tablespaces for everything under the
+	### specified schema.
+	###
+	def relations
+		return @relations if @relations
+		relations = {}
+
+		query =  %q{
+			SELECT c.oid AS oid,
+				c.relname AS name,
+				c.relkind AS kind,
+				t.spcname AS tspace
+			FROM pg_class AS c
+			LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
+			LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
+			WHERE c.relkind = 'r' }
+		query << "AND n.nspname='#{@opts.schema}'" if @opts.schema
+
+		# Get the relations list, along with each element's current tablespace.
+		#
+		self.db.exec( query ) do |res|
+			res.each do |row|
+				relations[ row['oid'] ] = {
+					:name       => row['name'],
+					:tablespace => row['tspace'],
+					:indexes    => [],
+					:parent     => nil
+				}
+			end
+		end
+
+		# Add table inheritance information.
+		#
+		db.exec 'SELECT inhrelid AS oid, inhparent AS parent FROM pg_inherits' do |res|
+			res.each do |row|
+				relations[ row['oid'] ][ :parent ] = row['parent']
+			end
+		end
+
+		# Remove tables that don't qualify for warehousing.
+		#
+		#   - Tables that are not children of a parent
+		#   - Tables that are already in the warehouse tablespace
+		#   - The currently active child (it's likely being written to!)
+		#   - Any table that can't be parsed into the specified format
+		#
+		relations.reject! do |oid, val|
+			begin
+				val[:parent].nil? ||
+				val[:tablespace] == @opts.tablespace ||
+				val[:name] == Time.now.strftime( @opts.format ) ||
+				! DateTime.strptime( val[:name], @opts.format )
+			rescue ArgumentError
+				true
+			end
+		end
+
+		query = %q{
+			SELECT c.oid AS oid,
+				i.indexname AS name
+			FROM pg_class AS c
+			INNER JOIN pg_indexes AS i
+				ON i.tablename = c.relname }
+		query << "AND i.schemaname='#{@opts.schema}'" if @opts.schema
+
+		# Attach index names to tables.
+		#
+		db.exec( query ) do |res|
+			res.each do |row|
+				relations[ row['oid'] ][ :indexes ] << row['name'] if relations[ row['oid'] ]
+			end
+		end
+
+		return relations
+	end
+
+
+	### Wrap arbitrary commands in a human readable timer.
+	###
+	def timer
+		start = Time.now
+		yield
+		age = Time.now - start
+
+		diff = age
+		secs = diff % 60
+		diff = ( diff - secs ) / 60
+		mins = diff % 60
+		diff = ( diff - mins ) / 60
+		hour = diff % 24
+
+		return "%02d:%02d:%02d" % [ hour, mins, secs ]
+	end
+end
+
+
+### Parse command line arguments.  Return a struct of global options.
+###
+def parse_args( args )
+	options          = OpenStruct.new
+	options.database = Etc.getpwuid( Process.uid ).name
+	options.host     = '127.0.0.1'
+	options.port     = 5432
+	options.user     = Etc.getpwuid( Process.uid ).name
+	options.sslmode  = 'prefer'
+	options.tablespace = 'warehouse'
+
+	opts = OptionParser.new do |opts|
+		opts.banner = "Usage: #{$0} [options]"
+
+		opts.separator ''
+		opts.separator 'Connection options:'
+
+		opts.on( '-d', '--database DBNAME',
+				"specify the database to connect to (default: \"#{options.database}\")" ) do |db|
+			options.database = db
+		end
+
+		opts.on( '-h', '--host HOSTNAME', 'database server host' ) do |host|
+			options.host = host
+		end
+
+		opts.on( '-p', '--port PORT', Integer,
+				"database server port (default: \"#{options.port}\")" ) do |port|
+			options.port = port
+		end
+
+		opts.on( '-n', '--schema SCHEMA', String,
+				"operate on the named schema only (default: none)" ) do |schema|
+			options.schema = schema
+		end
+
+		opts.on( '-T', '--tablespace SPACE', String,
+				"move old tables to this tablespace (default: \"#{options.tablespace}\")" ) do |tb|
+			options.tablespace = tb
+		end
+
+		opts.on( '-F', '--tableformat FORMAT', String,
+				"The naming format (strftime) for the inherited tables (default: none)" ) do |format|
+			options.format = format
+		end
+
+		opts.on( '-U', '--user NAME',
+				"database user name (default: \"#{options.user}\")" ) do |user|
+			options.user = user
+		end
+
+		opts.on( '-W', 'force password prompt' ) do |pw|
+			print 'Password: '
+			begin
+				system 'stty -echo'
+				options.pass = gets.chomp
+			ensure
+				system 'stty echo'
+				puts
+			end
+		end
+
+		opts.separator ''
+		opts.separator 'Other options:'
+
+		opts.on_tail( '--dry-run', "don't actually do anything" ) do
+			options.dryrun = true
+		end
+
+		opts.on_tail( '--help', 'show this help, then exit' ) do
+			$stderr.puts opts
+			exit
+		end
+
+		opts.on_tail( '--version', 'output version information, then exit' ) do
+			puts Stats::VERSION
+			exit
+		end
+	end
+
+	opts.parse!( args )
+	return options
+end
+
+
+if __FILE__ == $0
+	opts = parse_args( ARGV )
+	raise ArgumentError, "A naming format (-F) is required." unless opts.format
+
+	$stdout.sync = true
+	PGWarehouse.new( opts ).migrate
+end
+
+

data/vendor/database-extensions/install.sql

@@ -0,0 +1,317 @@
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+CREATE TYPE ore_64_8_v1_term AS (
+  bytes bytea
+);
+
+CREATE TYPE ore_64_8_v1 AS (
+  terms ore_64_8_v1_term[]
+);
+
+CREATE OR REPLACE FUNCTION compare_ore_64_8_v1_term(a ore_64_8_v1_term, b ore_64_8_v1_term) returns integer AS $$
+  DECLARE
+    eq boolean := true;
+    unequal_block smallint := 0;
+    hash_key bytea;
+    target_block bytea;
+
+    left_block_size CONSTANT smallint := 16;
+    right_block_size CONSTANT smallint := 32;
+    right_offset CONSTANT smallint := 136; -- 8 * 17
+
+    indicator smallint := 0;
+  BEGIN
+    IF a IS NULL AND b IS NULL THEN
+      RETURN 0;
+    END IF;
+
+    IF a IS NULL THEN
+      RETURN -1;
+    END IF;
+
+    IF b IS NULL THEN
+      RETURN 1;
+    END IF;
+
+    IF bit_length(a.bytes) != bit_length(b.bytes) THEN
+      RAISE EXCEPTION 'Ciphertexts are different lengths';
+    END IF;
+
+    FOR block IN 0..7 LOOP
+      -- Compare each PRP (byte from the first 8 bytes) and PRF block (8 byte
+      -- chunks of the rest of the value).
+      -- NOTE:
+      -- * Substr is ordinally indexed (hence 1 and not 0, and 9 and not 8).
+      -- * We are not worrying about timing attacks here; don't fret about
+      --   the OR or !=.
+      IF
+        substr(a.bytes, 1 + block, 1) != substr(b.bytes, 1 + block, 1)
+        OR substr(a.bytes, 9 + left_block_size * block, left_block_size) != substr(b.bytes, 9 + left_block_size * BLOCK, left_block_size)
+      THEN
+        -- set the first unequal block we find
+        IF eq THEN
+          unequal_block := block;
+        END IF;
+        eq = false;
+      END IF;
+    END LOOP;
+
+    IF eq THEN
+      RETURN 0::integer;
+    END IF;
+
+    -- Hash key is the IV from the right CT of b
+    hash_key := substr(b.bytes, right_offset + 1, 16);
+
+    -- first right block is at right offset + nonce_size (ordinally indexed)
+    target_block := substr(b.bytes, right_offset + 17 + (unequal_block * right_block_size), right_block_size);
+
+    indicator := (
+      get_bit(
+        encrypt(
+          substr(a.bytes, 9 + (left_block_size * unequal_block), left_block_size),
+          hash_key,
+          'aes-ecb'
+        ),
+        0
+      ) + get_bit(target_block, get_byte(a.bytes, unequal_block))) % 2;
+
+    IF indicator = 1 THEN
+      RETURN 1::integer;
+    ELSE
+      RETURN -1::integer;
+    END IF;
+  END;
+$$ LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_term_eq(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1_term(a, b) = 0
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_term_neq(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1_term(a, b) <> 0
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_term_lt(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1_term(a, b) = -1
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_term_lte(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1_term(a, b) != 1
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_term_gt(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1_term(a, b) = 1
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_term_gte(a ore_64_8_v1_term, b ore_64_8_v1_term) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1_term(a, b) != -1
+$$ LANGUAGE SQL;
+
+CREATE OPERATOR = (
+  PROCEDURE="ore_64_8_v1_term_eq",
+  LEFTARG=ore_64_8_v1_term,
+  RIGHTARG=ore_64_8_v1_term,
+  NEGATOR = <>,
+  RESTRICT = eqsel,
+  JOIN = eqjoinsel,
+  HASHES,
+  MERGES
+);
+
+CREATE OPERATOR <> (
+  PROCEDURE="ore_64_8_v1_term_neq",
+  LEFTARG=ore_64_8_v1_term,
+  RIGHTARG=ore_64_8_v1_term,
+  NEGATOR = =,
+  RESTRICT = eqsel,
+  JOIN = eqjoinsel,
+  HASHES,
+  MERGES
+);
+
+CREATE OPERATOR > (
+  PROCEDURE="ore_64_8_v1_term_gt",
+  LEFTARG=ore_64_8_v1_term,
+  RIGHTARG=ore_64_8_v1_term,
+  COMMUTATOR = <,
+  NEGATOR = <=,
+  RESTRICT = scalargtsel,
+  JOIN = scalargtjoinsel
+);
+
+CREATE OPERATOR < (
+  PROCEDURE="ore_64_8_v1_term_lt",
+  LEFTARG=ore_64_8_v1_term,
+  RIGHTARG=ore_64_8_v1_term,
+  COMMUTATOR = >,
+  NEGATOR = >=,
+  RESTRICT = scalarltsel,
+  JOIN = scalarltjoinsel
+);
+
+CREATE OPERATOR <= (
+  PROCEDURE="ore_64_8_v1_term_lte",
+  LEFTARG=ore_64_8_v1_term,
+  RIGHTARG=ore_64_8_v1_term,
+  COMMUTATOR = >=,
+  NEGATOR = >,
+  RESTRICT = scalarlesel,
+  JOIN = scalarlejoinsel
+);
+
+CREATE OPERATOR >= (
+  PROCEDURE="ore_64_8_v1_term_gte",
+  LEFTARG=ore_64_8_v1_term,
+  RIGHTARG=ore_64_8_v1_term,
+  COMMUTATOR = <=,
+  NEGATOR = <,
+  RESTRICT = scalarlesel,
+  JOIN = scalarlejoinsel
+);
+
+CREATE OPERATOR FAMILY ore_64_8_v1_term_btree_ops USING btree;
+CREATE OPERATOR CLASS ore_64_8_v1_term_btree_ops DEFAULT FOR TYPE ore_64_8_v1_term USING btree FAMILY ore_64_8_v1_term_btree_ops  AS
+        OPERATOR 1 <,
+        OPERATOR 2 <=,
+        OPERATOR 3 =,
+        OPERATOR 4 >=,
+        OPERATOR 5 >,
+        FUNCTION 1 compare_ore_64_8_v1_term(a ore_64_8_v1_term, b ore_64_8_v1_term);
+
+-- Compare the "head" of each array and recurse if necessary
+-- This function assumes an empty string is "less than" everything else
+-- so if a is empty we return -1, if be is empty and a isn't, we return 1.
+-- If both are empty we return 0. This cases probably isn't necessary as equality
+-- doesn't always make sense but it's here for completeness.
+-- If both are non-empty, we compare the first element. If they are equal
+-- we need to consider the next block so we recurse, otherwise we return the comparison result.
+CREATE OR REPLACE FUNCTION compare_ore_array(a ore_64_8_v1_term[], b ore_64_8_v1_term[]) returns integer AS $$
+  DECLARE
+    cmp_result integer;
+  BEGIN
+    IF (array_length(a, 1) = 0 OR a IS NULL) AND (array_length(b, 1) = 0 OR b IS NULL) THEN
+      RETURN 0;
+    END IF;
+    IF array_length(a, 1) = 0 OR a IS NULL THEN
+      RETURN -1;
+    END IF;
+    IF array_length(b, 1) = 0 OR a IS NULL THEN
+      RETURN 1;
+    END IF;
+
+    cmp_result := compare_ore_64_8_v1_term(a[1], b[1]);
+    IF cmp_result = 0 THEN
+    -- Removes the first element in the array, and calls this fn again to compare the next element/s in the array.
+      RETURN compare_ore_array(a[2:array_length(a,1)], b[2:array_length(b,1)]);
+    END IF;
+
+    RETURN cmp_result;
+  END
+$$ LANGUAGE plpgsql;
+
+-- This function uses lexicographic comparison
+CREATE OR REPLACE FUNCTION compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1) returns integer AS $$
+  DECLARE
+    cmp_result integer;
+  BEGIN
+    -- Recursively compare blocks bailing as soon as we can make a decision
+    RETURN compare_ore_array(a.terms, b.terms);
+  END
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_eq(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1(a, b) = 0
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_neq(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1(a, b) <> 0
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_lt(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1(a, b) = -1
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_lte(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1(a, b) != 1
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_gt(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1(a, b) = 1
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION ore_64_8_v1_gte(a ore_64_8_v1, b ore_64_8_v1) RETURNS boolean AS $$
+  SELECT compare_ore_64_8_v1(a, b) != -1
+$$ LANGUAGE SQL;
+
+CREATE OPERATOR = (
+  PROCEDURE="ore_64_8_v1_eq",
+  LEFTARG=ore_64_8_v1,
+  RIGHTARG=ore_64_8_v1,
+  NEGATOR = <>,
+  RESTRICT = eqsel,
+  JOIN = eqjoinsel,
+  HASHES,
+  MERGES
+);
+
+CREATE OPERATOR <> (
+  PROCEDURE="ore_64_8_v1_neq",
+  LEFTARG=ore_64_8_v1,
+  RIGHTARG=ore_64_8_v1,
+  NEGATOR = =,
+  RESTRICT = eqsel,
+  JOIN = eqjoinsel,
+  HASHES,
+  MERGES
+);
+
+CREATE OPERATOR > (
+  PROCEDURE="ore_64_8_v1_gt",
+  LEFTARG=ore_64_8_v1,
+  RIGHTARG=ore_64_8_v1,
+  COMMUTATOR = <,
+  NEGATOR = <=,
+  RESTRICT = scalargtsel,
+  JOIN = scalargtjoinsel
+);
+
+CREATE OPERATOR < (
+  PROCEDURE="ore_64_8_v1_lt",
+  LEFTARG=ore_64_8_v1,
+  RIGHTARG=ore_64_8_v1,
+  COMMUTATOR = >,
+  NEGATOR = >=,
+  RESTRICT = scalarltsel,
+  JOIN = scalarltjoinsel
+);
+
+CREATE OPERATOR <= (
+  PROCEDURE="ore_64_8_v1_lte",
+  LEFTARG=ore_64_8_v1,
+  RIGHTARG=ore_64_8_v1,
+  COMMUTATOR = >=,
+  NEGATOR = >,
+  RESTRICT = scalarlesel,
+  JOIN = scalarlejoinsel
+);
+
+CREATE OPERATOR >= (
+  PROCEDURE="ore_64_8_v1_gte",
+  LEFTARG=ore_64_8_v1,
+  RIGHTARG=ore_64_8_v1,
+  COMMUTATOR = <=,
+  NEGATOR = <,
+  RESTRICT = scalarlesel,
+  JOIN = scalarlejoinsel
+);
+
+CREATE OPERATOR FAMILY ore_64_8_v1_btree_ops USING btree;
+CREATE OPERATOR CLASS ore_64_8_v1_btree_ops DEFAULT FOR TYPE ore_64_8_v1 USING btree FAMILY ore_64_8_v1_btree_ops  AS
+        OPERATOR 1 <,
+        OPERATOR 2 <=,
+        OPERATOR 3 =,
+        OPERATOR 4 >=,
+        OPERATOR 5 >,
+        FUNCTION 1 compare_ore_64_8_v1(a ore_64_8_v1, b ore_64_8_v1);

data/vendor/database-extensions/uninstall.sql

@@ -0,0 +1,20 @@
+-- TODO: what happens if we try to uninstall a type which is in use?
+DROP OPERATOR IF EXISTS = (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE;
+DROP OPERATOR IF EXISTS <> (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE;
+DROP OPERATOR IF EXISTS > (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE;
+DROP OPERATOR IF EXISTS < (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE;
+DROP OPERATOR IF EXISTS <= (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE;
+DROP OPERATOR IF EXISTS >= (ore_64_8_v1_term, ore_64_8_v1_term) CASCADE;
+DROP OPERATOR CLASS IF EXISTS ore_64_8_v1_term_btree_ops USING btree CASCADE;
+DROP OPERATOR FAMILY IF EXISTS ore_64_8_v1_term_btree_ops USING btree CASCADE;
+DROP TYPE IF EXISTS ore_64_8_v1_term CASCADE;
+
+DROP OPERATOR IF EXISTS = (ore_64_8_v1, ore_64_8_v1) CASCADE;
+DROP OPERATOR IF EXISTS <> (ore_64_8_v1, ore_64_8_v1) CASCADE;
+DROP OPERATOR IF EXISTS > (ore_64_8_v1, ore_64_8_v1) CASCADE;
+DROP OPERATOR IF EXISTS < (ore_64_8_v1, ore_64_8_v1) CASCADE;
+DROP OPERATOR IF EXISTS <= (ore_64_8_v1, ore_64_8_v1) CASCADE;
+DROP OPERATOR IF EXISTS >= (ore_64_8_v1, ore_64_8_v1) CASCADE;
+DROP OPERATOR CLASS IF EXISTS ore_64_8_v1_btree_ops USING btree CASCADE;
+DROP OPERATOR FAMILY IF EXISTS ore_64_8_v1_btree_ops USING btree CASCADE;
+DROP TYPE IF EXISTS ore_64_8_v1 CASCADE;