repertoire-faceting 0.5.5 → 0.6.0

data/ext/bytea/bytea.sql

@@ -0,0 +1,173 @@
+-- ============================================================================
+-- Faceting API implementing bitmap indices using PostgreSQL's built-in BYTEA
+-- type, processed using plv8 typed arrays.
+--
+-- This API is suitable for deployment on Heroku, where plv8 is installed by
+-- default. Performance is many times better than the VARBIT-based faceting
+-- API, primarily because of optimisations in memory handling in the count
+-- function.
+--
+-- See https://code.google.com/p/plv8js/wiki/PLV8
+--     https://postgres.heroku.com/blog/past/2013/6/5/javascript_in_your_postgres/
+--
+-- Christopher York
+-- MIT Hyperstudio
+-- February 2014
+-- ============================================================================
+
+CREATE EXTENSION IF NOT EXISTS plv8;
+
+SET bytea_output TO hex;
+
+-- these functions are in pl/pgsql, because they involve appending bytea values,
+-- which is easier done with direct access to the || operator
+
+CREATE FUNCTION @extschema@.sig_resize( sig BYTEA, bits INT ) RETURNS BYTEA AS $$
+DECLARE
+  len INT;
+  bytes INT;
+BEGIN
+  bytes := ceil(bits / 8.0)::INT;
+  len := length(sig);
+  IF bytes > len THEN
+    -- RAISE NOTICE 'Extending signature from % to % bytes', len, bytes;
+    RETURN sig || ('\x' || repeat('00', bytes - len))::BYTEA;
+  ELSIF bits < len THEN
+    -- no provision in PostgreSQL for truncating a BYTEA
+    RETURN sig;
+  END IF;
+  RETURN sig;
+END $$ LANGUAGE plpgsql STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_set( sig BYTEA, pos INT, val INT) RETURNS BYTEA AS $$
+BEGIN
+  RETURN set_bit(sig_resize(sig, pos+1), pos, val);
+END $$ LANGUAGE plpgsql STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_set( sig BYTEA, pos INT) RETURNS BYTEA AS $$
+BEGIN
+  RETURN @extschema@.sig_set(sig, pos, 1);
+END $$ LANGUAGE plpgsql STRICT IMMUTABLE;
+
+-- these functions are in javascript, because (1) pl/pgsql is close
+-- to the worst language in the world; (2) plv8's typed arrays make 
+-- the count function much faster
+
+CREATE FUNCTION @extschema@.sig_get( sig BYTEA, pos INT ) RETURNS INT AS $$
+  if (pos <= sig.length * 8) {
+    return sig[ Math.floor(pos / 8) ] >> (pos % 8) & 1;
+  } else {
+    return 0;
+  }
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_length( sig BYTEA ) RETURNS INT AS $$
+  return sig.length;
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_and(sig1 BYTEA, sig2 BYTEA) RETURNS BYTEA AS $$
+  if (sig2.length < sig1.length) {
+    var tmp = sig1;
+    sig1 = sig2;
+    sig2 = tmp;
+  }
+  for (var i = 0; i < sig1.length; i++) {
+    sig1[i] = sig1[i] & sig2[i];
+  }
+  return sig1;
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_or(sig1 BYTEA, sig2 BYTEA) RETURNS BYTEA AS $$
+  if (sig2.length > sig1.length) {
+    var tmp = sig1;
+    sig1 = sig2;
+    sig2 = tmp;
+  }
+  for (var i = 0; i < sig2.length; i++) {
+    sig1[i] = sig1[i] | sig2[i];
+  }
+  return sig1;
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.count(sig bytea) RETURNS int4 AS $$
+  var count_table = [
+    0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8,
+  ];
+  var count = 0; 
+  for (var i = 0; i < sig.length; i++) { count += count_table[ sig[i] ]; } 
+  return count;
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.contains( sig BYTEA, pos INT ) RETURNS BOOL AS $$
+  return (pos <= sig.length * 8) && (sig[ Math.floor(pos / 8) ] >> (pos % 8) & 1);
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.members( sig BYTEA ) RETURNS SETOF INT AS $$
+  for (var i = 0; i < sig.length; i++) {
+    for (var j = 0; j < 8; j++) {
+      if (sig[i] >> j & 1) {
+        plv8.return_next(i * 8 + j);
+      }
+    }
+  }
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+
+-- operators for faceting
+
+CREATE OPERATOR @extschema@.& (
+    leftarg = BYTEA,
+    rightarg = BYTEA,
+    procedure = @extschema@.sig_and,
+    commutator = &
+);
+
+CREATE OPERATOR @extschema@.| (
+    leftarg = BYTEA,
+    rightarg = BYTEA,
+    procedure = @extschema@.sig_or,
+    commutator = |
+);
+
+CREATE OPERATOR @extschema@.+ (
+    leftarg = BYTEA,
+    rightarg = int,
+    procedure = @extschema@.sig_set
+);
+
+
+-- aggregate functions for faceting
+
+CREATE AGGREGATE @extschema@.collect( BYTEA )
+(
+	sfunc = @extschema@.sig_or,
+	stype = BYTEA
+);
+
+CREATE AGGREGATE @extschema@.filter( BYTEA )
+(
+   sfunc = @extschema@.sig_and,
+   stype = BYTEA
+);
+
+CREATE AGGREGATE @extschema@.signature( INT )
+(
+	sfunc = @extschema@.sig_set,
+  stype = BYTEA,
+  initcond = ''
+);

data/ext/bytea/faceting_bytea.control

@@ -0,0 +1,6 @@
+# Faceting PostgreSQL extension module
+ 
+comment = 'API for faceted indexing and queries (based on plv8 + bytea bitmaps)'
+requires = 'plv8, plpgsql'
+default_version = '0.6.0'
+schema = 'facet'

data/ext/common/util.sql

@@ -0,0 +1,35 @@
+-- ============================================================================
+-- These functions are common to all bindings of the Repertoire faceting API
+--
+-- Christopher York
+-- MIT Hyperstudio
+-- February 2014
+-- ============================================================================
+
+
+-- Aggregator to measure how many bits from a loosely-packed id column would be wasted, if
+-- they were all collected into a bitset signature. Returns a float between 0 (no waste)
+-- and 1.0 (all waste). An example of its use:
+--
+-- SELECT wastage(id) FROM nobelists
+--  =# 0.999999
+--
+-- ALTER TABLE nobelists ADD COLUMN _packed_id SERIAL
+-- SELECT wastage(_packed_id) FROM nobelists
+--  =# 0.015625
+--
+CREATE FUNCTION @extschema@.wastage_proportion(state INT[]) RETURNS double precision AS $$
+  SELECT (1.0 - (state[1]::double precision / (COALESCE(state[2], 0.0) + 1.0)))
+$$ LANGUAGE sql;
+
+CREATE FUNCTION @extschema@.wastage_accum(state INT[], val INT) RETURNS INT[] AS $$
+  SELECT ARRAY[ state[1]+1, GREATEST(state[2], val) ];
+$$ LANGUAGE sql;
+
+CREATE AGGREGATE @extschema@.wastage( INT )
+(
+	sfunc = @extschema@.wastage_accum,
+	stype = INT[],
+	finalfunc = @extschema@.wastage_proportion,
+	initcond = '{0,0}'
+);

data/ext/faceting--0.6.0.sql

@@ -0,0 +1,251 @@
+-- ============================================================================
+-- Faceting API implementing bitmap indices using a custom C datatype and
+-- associated functions.
+--
+-- This API is to be preferred in all situations where it is possible to
+-- build and install the datatype (requires superuser access to PostgreSQL)
+--
+-- Christopher York
+-- MIT Hyperstudio
+-- February 2014
+-- ============================================================================
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION faceting" to load this the default faceting API.\quit
+
+-- functions for bitmap indices using datatype written in C
+
+CREATE TYPE @extschema@.signature;
+
+-- basic i/o functions for signatures
+
+CREATE FUNCTION @extschema@.sig_in(cstring)
+  RETURNS signature
+  AS 'signature.so', 'sig_in'
+  LANGUAGE C STRICT;
+
+CREATE FUNCTION @extschema@.sig_out(signature)
+  RETURNS cstring
+  AS 'signature.so', 'sig_out'
+  LANGUAGE C STRICT;
+
+-- signature postgresql type
+
+CREATE TYPE @extschema@.signature (
+	INTERNALLENGTH = VARIABLE,
+	INPUT = sig_in,
+	OUTPUT = sig_out,
+	STORAGE = extended
+);
+
+-- functions for signatures
+
+CREATE FUNCTION @extschema@.sig_resize( signature, INT )
+  RETURNS signature
+  AS 'signature.so', 'sig_resize'
+  LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_set( signature, INT, INT )
+  RETURNS signature
+  AS 'signature.so', 'sig_set'
+  LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_set( signature, INT )
+  RETURNS signature
+  AS 'signature.so', 'sig_set'
+  LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_get( signature, INT )
+  RETURNS INT
+  AS 'signature.so', 'sig_get'
+  LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_length( signature )
+	RETURNS INT
+	AS 'signature.so', 'sig_length'
+	LANGUAGE C STRICT IMMUTABLE;
+	
+CREATE FUNCTION @extschema@.sig_min( signature )
+	RETURNS INT
+	AS 'signature.so', 'sig_min'
+	LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_and( signature, signature )
+  RETURNS signature
+  AS 'signature.so', 'sig_and'
+  LANGUAGE C STRICT IMMUTABLE;	
+
+CREATE FUNCTION @extschema@.sig_or( signature, signature )
+  RETURNS signature
+  AS 'signature.so', 'sig_or'
+  LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_xor( signature )
+  RETURNS signature
+  AS 'signature.so', 'sig_xor'
+  LANGUAGE C STRICT IMMUTABLE;
+ 
+CREATE FUNCTION @extschema@.count( signature )
+	RETURNS INT
+	AS 'signature.so', 'count'
+	LANGUAGE C STRICT IMMUTABLE;
+	
+CREATE FUNCTION @extschema@.contains( signature, INT )
+  RETURNS BOOL
+  AS 'signature.so', 'contains'
+  LANGUAGE C STRICT IMMUTABLE;	
+	
+CREATE FUNCTION @extschema@.members( signature )
+  RETURNS SETOF INT
+  AS 'signature.so', 'members'
+  LANGUAGE C STRICT IMMUTABLE;
+  
+CREATE FUNCTION @extschema@.sig_cmp( signature, signature )
+  RETURNS INT
+  AS 'signature.so', 'sig_cmp'
+  LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_lt( signature, signature )
+  RETURNS BOOL
+  AS 'signature.so', 'sig_lt'
+  LANGUAGE C STRICT IMMUTABLE;
+ 
+CREATE FUNCTION @extschema@.sig_lte( signature, signature )
+  RETURNS BOOL
+  AS 'signature.so', 'sig_lte'
+  LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_eq( signature, signature )
+  RETURNS BOOL
+  AS 'signature.so', 'sig_eq'
+  LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_gt( signature, signature )
+  RETURNS BOOL
+  AS 'signature.so', 'sig_gt'
+  LANGUAGE C STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_gte( signature, signature )
+  RETURNS BOOL
+  AS 'signature.so', 'sig_gte'
+  LANGUAGE C STRICT IMMUTABLE;
+
+
+-- operators for signatures
+
+CREATE OPERATOR @extschema@.& (
+    leftarg = signature,
+    rightarg = signature,
+    procedure = @extschema@.sig_and,
+    commutator = &
+);
+
+CREATE OPERATOR @extschema@.| (
+    leftarg = signature,
+    rightarg = signature,
+    procedure = @extschema@.sig_or,
+    commutator = |
+);
+
+CREATE OPERATOR @extschema@.+ (
+    leftarg = signature,
+    rightarg = int,
+    procedure = @extschema@.sig_set
+);
+ 
+CREATE OPERATOR @extschema@.< (
+   leftarg = signature, rightarg = signature, procedure = sig_lt,
+   commutator = > , negator = >= ,
+   restrict = scalarltsel, join = scalarltjoinsel
+);
+
+CREATE OPERATOR @extschema@.<= (
+   leftarg = signature, rightarg = signature, procedure = sig_lte,
+   commutator = >= , negator = > ,
+   restrict = scalarltsel, join = scalarltjoinsel
+);
+
+CREATE OPERATOR @extschema@.= (
+   leftarg = signature, rightarg = signature, procedure = sig_eq,
+   commutator = = , negator = <> ,
+   restrict = eqsel, join = eqjoinsel
+);
+
+CREATE OPERATOR >= (
+   leftarg = signature, rightarg = signature, procedure = sig_gte,
+   commutator = <= , negator = < ,
+   restrict = scalargtsel, join = scalargtjoinsel
+);
+
+CREATE OPERATOR @extschema@.> (
+   leftarg = signature, rightarg = signature, procedure = sig_gt,
+   commutator = < , negator = <= ,
+   restrict = scalargtsel, join = scalargtjoinsel
+);
+
+-- index operator classes for signatures
+
+CREATE OPERATOR CLASS @extschema@.signature_ops
+    DEFAULT FOR TYPE signature USING btree AS
+        OPERATOR        1       < ,
+        OPERATOR        2       <= ,
+        OPERATOR        3       = ,
+        OPERATOR        4       >= ,
+        OPERATOR        5       > ,
+        FUNCTION        1       sig_cmp(signature, signature);
+
+
+-- aggregate functions for faceting
+
+CREATE AGGREGATE @extschema@.collect( signature )
+(
+	sfunc = @extschema@.sig_or,
+	stype = signature
+);
+
+CREATE AGGREGATE @extschema@.filter( signature )
+(
+   sfunc = @extschema@.sig_and,
+   stype = signature
+);
+
+CREATE AGGREGATE @extschema@.signature( INT )
+(
+	sfunc = @extschema@.sig_set,
+	stype = signature,
+  initcond = '0'
+);-- ============================================================================
+-- These functions are common to all bindings of the Repertoire faceting API
+--
+-- Christopher York
+-- MIT Hyperstudio
+-- February 2014
+-- ============================================================================
+
+
+-- Aggregator to measure how many bits from a loosely-packed id column would be wasted, if
+-- they were all collected into a bitset signature. Returns a float between 0 (no waste)
+-- and 1.0 (all waste). An example of its use:
+--
+-- SELECT wastage(id) FROM nobelists
+--  =# 0.999999
+--
+-- ALTER TABLE nobelists ADD COLUMN _packed_id SERIAL
+-- SELECT wastage(_packed_id) FROM nobelists
+--  =# 0.015625
+--
+CREATE FUNCTION @extschema@.wastage_proportion(state INT[]) RETURNS double precision AS $$
+  SELECT (1.0 - (state[1]::double precision / (COALESCE(state[2], 0.0) + 1.0)))
+$$ LANGUAGE sql;
+
+CREATE FUNCTION @extschema@.wastage_accum(state INT[], val INT) RETURNS INT[] AS $$
+  SELECT ARRAY[ state[1]+1, GREATEST(state[2], val) ];
+$$ LANGUAGE sql;
+
+CREATE AGGREGATE @extschema@.wastage( INT )
+(
+	sfunc = @extschema@.wastage_accum,
+	stype = INT[],
+	finalfunc = @extschema@.wastage_proportion,
+	initcond = '{0,0}'
+);

data/ext/faceting_bytea--0.6.0.sql

@@ -0,0 +1,207 @@
+-- ============================================================================
+-- Faceting API implementing bitmap indices using PostgreSQL's built-in BYTEA
+-- type, processed using plv8 typed arrays.
+--
+-- This API is suitable for deployment on Heroku, where plv8 is installed by
+-- default. Performance is many times better than the VARBIT-based faceting
+-- API, primarily because of optimisations in memory handling in the count
+-- function.
+--
+-- See https://code.google.com/p/plv8js/wiki/PLV8
+--     https://postgres.heroku.com/blog/past/2013/6/5/javascript_in_your_postgres/
+--
+-- Christopher York
+-- MIT Hyperstudio
+-- February 2014
+-- ============================================================================
+
+CREATE EXTENSION IF NOT EXISTS plv8;
+
+SET bytea_output TO hex;
+
+-- these functions are in pl/pgsql, because they involve appending bytea values,
+-- which is easier done with direct access to the || operator
+
+CREATE FUNCTION @extschema@.sig_resize( sig BYTEA, bits INT ) RETURNS BYTEA AS $$
+DECLARE
+  len INT;
+  bytes INT;
+BEGIN
+  bytes := ceil(bits / 8.0)::INT;
+  len := length(sig);
+  IF bytes > len THEN
+    -- RAISE NOTICE 'Extending signature from % to % bytes', len, bytes;
+    RETURN sig || ('\x' || repeat('00', bytes - len))::BYTEA;
+  ELSIF bits < len THEN
+    -- no provision in PostgreSQL for truncating a BYTEA
+    RETURN sig;
+  END IF;
+  RETURN sig;
+END $$ LANGUAGE plpgsql STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_set( sig BYTEA, pos INT, val INT) RETURNS BYTEA AS $$
+BEGIN
+  RETURN set_bit(sig_resize(sig, pos+1), pos, val);
+END $$ LANGUAGE plpgsql STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_set( sig BYTEA, pos INT) RETURNS BYTEA AS $$
+BEGIN
+  RETURN @extschema@.sig_set(sig, pos, 1);
+END $$ LANGUAGE plpgsql STRICT IMMUTABLE;
+
+-- these functions are in javascript, because (1) pl/pgsql is close
+-- to the worst language in the world; (2) plv8's typed arrays make 
+-- the count function much faster
+
+CREATE FUNCTION @extschema@.sig_get( sig BYTEA, pos INT ) RETURNS INT AS $$
+  if (pos <= sig.length * 8) {
+    return sig[ Math.floor(pos / 8) ] >> (pos % 8) & 1;
+  } else {
+    return 0;
+  }
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_length( sig BYTEA ) RETURNS INT AS $$
+  return sig.length;
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_and(sig1 BYTEA, sig2 BYTEA) RETURNS BYTEA AS $$
+  if (sig2.length < sig1.length) {
+    var tmp = sig1;
+    sig1 = sig2;
+    sig2 = tmp;
+  }
+  for (var i = 0; i < sig1.length; i++) {
+    sig1[i] = sig1[i] & sig2[i];
+  }
+  return sig1;
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.sig_or(sig1 BYTEA, sig2 BYTEA) RETURNS BYTEA AS $$
+  if (sig2.length > sig1.length) {
+    var tmp = sig1;
+    sig1 = sig2;
+    sig2 = tmp;
+  }
+  for (var i = 0; i < sig2.length; i++) {
+    sig1[i] = sig1[i] | sig2[i];
+  }
+  return sig1;
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.count(sig bytea) RETURNS int4 AS $$
+  var count_table = [
+    0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+    4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8,
+  ];
+  var count = 0; 
+  for (var i = 0; i < sig.length; i++) { count += count_table[ sig[i] ]; } 
+  return count;
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.contains( sig BYTEA, pos INT ) RETURNS BOOL AS $$
+  return (pos <= sig.length * 8) && (sig[ Math.floor(pos / 8) ] >> (pos % 8) & 1);
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+CREATE FUNCTION @extschema@.members( sig BYTEA ) RETURNS SETOF INT AS $$
+  for (var i = 0; i < sig.length; i++) {
+    for (var j = 0; j < 8; j++) {
+      if (sig[i] >> j & 1) {
+        plv8.return_next(i * 8 + j);
+      }
+    }
+  }
+$$ LANGUAGE plv8 STRICT IMMUTABLE;
+
+
+-- operators for faceting
+
+CREATE OPERATOR @extschema@.& (
+    leftarg = BYTEA,
+    rightarg = BYTEA,
+    procedure = @extschema@.sig_and,
+    commutator = &
+);
+
+CREATE OPERATOR @extschema@.| (
+    leftarg = BYTEA,
+    rightarg = BYTEA,
+    procedure = @extschema@.sig_or,
+    commutator = |
+);
+
+CREATE OPERATOR @extschema@.+ (
+    leftarg = BYTEA,
+    rightarg = int,
+    procedure = @extschema@.sig_set
+);
+
+
+-- aggregate functions for faceting
+
+CREATE AGGREGATE @extschema@.collect( BYTEA )
+(
+	sfunc = @extschema@.sig_or,
+	stype = BYTEA
+);
+
+CREATE AGGREGATE @extschema@.filter( BYTEA )
+(
+   sfunc = @extschema@.sig_and,
+   stype = BYTEA
+);
+
+CREATE AGGREGATE @extschema@.signature( INT )
+(
+	sfunc = @extschema@.sig_set,
+  stype = BYTEA,
+  initcond = ''
+);-- ============================================================================
+-- These functions are common to all bindings of the Repertoire faceting API
+--
+-- Christopher York
+-- MIT Hyperstudio
+-- February 2014
+-- ============================================================================
+
+
+-- Aggregator to measure how many bits from a loosely-packed id column would be wasted, if
+-- they were all collected into a bitset signature. Returns a float between 0 (no waste)
+-- and 1.0 (all waste). An example of its use:
+--
+-- SELECT wastage(id) FROM nobelists
+--  =# 0.999999
+--
+-- ALTER TABLE nobelists ADD COLUMN _packed_id SERIAL
+-- SELECT wastage(_packed_id) FROM nobelists
+--  =# 0.015625
+--
+CREATE FUNCTION @extschema@.wastage_proportion(state INT[]) RETURNS double precision AS $$
+  SELECT (1.0 - (state[1]::double precision / (COALESCE(state[2], 0.0) + 1.0)))
+$$ LANGUAGE sql;
+
+CREATE FUNCTION @extschema@.wastage_accum(state INT[], val INT) RETURNS INT[] AS $$
+  SELECT ARRAY[ state[1]+1, GREATEST(state[2], val) ];
+$$ LANGUAGE sql;
+
+CREATE AGGREGATE @extschema@.wastage( INT )
+(
+	sfunc = @extschema@.wastage_accum,
+	stype = INT[],
+	finalfunc = @extschema@.wastage_proportion,
+	initcond = '{0,0}'
+);