pg 1.0.0 → 1.1.0.pre20180730144600

data/ext/pg_copy_coder.c

@@ -354,7 +354,7 @@ GetDecimalFromHex(char hex)
  * src/backend/commands/copy.c
  */
 static VALUE
-pg_text_dec_copy_row(t_pg_coder *conv, char *input_line, int len, int _tuple, int _field, int enc_idx)
+pg_text_dec_copy_row(t_pg_coder *conv, const char *input_line, int len, int _tuple, int _field, int enc_idx)
 {
 	t_pg_copycoder *this = (t_pg_copycoder *)conv;
 
@@ -368,8 +368,8 @@ pg_text_dec_copy_row(t_pg_coder *conv, char *input_line, int len, int _tuple, in
 	int fieldno;
 	int expected_fields;
 	char *output_ptr;
-	char *cur_ptr;
-	char *line_end_ptr;
+	const char *cur_ptr;
+	const char *line_end_ptr;
 	char *end_capa_ptr;
 	t_typemap *p_typemap;
 
@@ -392,8 +392,8 @@ pg_text_dec_copy_row(t_pg_coder *conv, char *input_line, int len, int _tuple, in
 	for (;;)
 	{
 		int found_delim = 0;
-		char *start_ptr;
-		char *end_ptr;
+		const char *start_ptr;
+		const char *end_ptr;
 		int input_len;
 
 		/* Remember start of field on input side */
@@ -585,7 +585,12 @@ init_pg_copycoder()
 	/* rb_mPG_TextEncoder = rb_define_module_under( rb_mPG, "TextEncoder" ); */
 	/* dummy = rb_define_class_under( rb_mPG_TextEncoder, "CopyRow", rb_cPG_CopyEncoder ); */
 	pg_define_coder( "CopyRow", pg_text_enc_copy_row, rb_cPG_CopyEncoder, rb_mPG_TextEncoder );
+	rb_include_module( rb_cPG_CopyEncoder, rb_mPG_BinaryFormatting );
+
 	/* rb_mPG_TextDecoder = rb_define_module_under( rb_mPG, "TextDecoder" ); */
 	/* dummy = rb_define_class_under( rb_mPG_TextDecoder, "CopyRow", rb_cPG_CopyDecoder ); */
 	pg_define_coder( "CopyRow", pg_text_dec_copy_row, rb_cPG_CopyDecoder, rb_mPG_TextDecoder );
+	/* Although CopyRow is a text decoder, data can contain zero bytes and are not zero terminated.
+	 * They are handled like binaries. So format is set to 1 (binary). */
+	rb_include_module( rb_cPG_CopyDecoder, rb_mPG_BinaryFormatting );
 }

data/ext/pg_result.c

@@ -1,6 +1,6 @@
 /*
  * pg_result.c - PG::Result class extension
- * $Id: pg_result.c,v 7c5d9608349f 2018/01/03 16:11:52 lars $
+ * $Id$
  *
  */
 
@@ -16,6 +16,102 @@ static t_pg_result *pgresult_get_this( VALUE );
 static t_pg_result *pgresult_get_this_safe( VALUE );
 
 
+#define PGRESULT_DATA_BLOCKSIZE 2048
+typedef struct pgresAttValue
+{
+	int			len;			/* length in bytes of the value */
+	char	   *value;			/* actual value, plus terminating zero byte */
+} PGresAttValue;
+
+
+static int
+count_leading_zero_bits(unsigned int x)
+{
+#if defined(__GNUC__) || defined(__clang__)
+	return __builtin_clz(x);
+#elif defined(_MSC_VER)
+	DWORD r = 0;
+	_BitScanForward(&r, x);
+	return (int)r;
+#else
+	unsigned int a;
+	for(a=0; a < sizeof(unsigned int) * 8; a++){
+		if( x & (1 << (sizeof(unsigned int) * 8 - 1))) return a;
+		x <<= 1;
+	}
+	return a;
+#endif
+}
+
+static ssize_t
+pgresult_approx_size(PGresult *result)
+{
+	int num_fields = PQnfields(result);
+	ssize_t size = 0;
+
+	if( num_fields > 0 ){
+		int num_tuples = PQntuples(result);
+
+		if( num_tuples > 0 ){
+			int pos;
+
+			/* This is a simple heuristic to determine the number of sample fields and subsequently to approximate the memory size taken by all field values of the result set.
+			 * Since scanning of all field values is would have a severe performance impact, only a small subset of fields is retrieved and the result is extrapolated to the whole result set.
+			 * The given algorithm has no real scientific background, but is made for speed and typical table layouts.
+			 */
+			int num_samples =
+				(num_fields < 9 ? num_fields : 39 - count_leading_zero_bits(num_fields-8)) *
+				(num_tuples < 8 ? 1 : 30 - count_leading_zero_bits(num_tuples));
+
+			/* start with scanning very last fields, since they are most probably in the cache */
+			for( pos = 0; pos < (num_samples+1)/2; pos++ ){
+				size += PQgetlength(result, num_tuples - 1 - (pos / num_fields), num_fields - 1 - (pos % num_fields));
+			}
+			/* scan the very first fields */
+			for( pos = 0; pos < num_samples/2; pos++ ){
+				size += PQgetlength(result, pos / num_fields, pos % num_fields);
+			}
+			/* extrapolate sample size to whole result set */
+			size = size * num_tuples * num_fields / num_samples;
+		}
+
+		/* count metadata */
+		size += num_fields * (
+				sizeof(PGresAttDesc) + /* column description */
+				num_tuples * (
+					sizeof(PGresAttValue) + 1 /* ptr, len and zero termination of each value */
+				)
+		);
+
+		/* Account free space due to libpq's default block size */
+		size = (size + PGRESULT_DATA_BLOCKSIZE - 1) / PGRESULT_DATA_BLOCKSIZE * PGRESULT_DATA_BLOCKSIZE;
+
+		/* count tuple pointers */
+		size += sizeof(void*) * ((num_tuples + 128 - 1) / 128 * 128);
+	}
+
+	size += 216; /* add PGresult size */
+
+	return size;
+}
+
+static void
+pgresult_clear( t_pg_result *this )
+{
+	if( this->pgresult && !this->autoclear ){
+		PQclear(this->pgresult);
+#ifdef HAVE_RB_GC_ADJUST_MEMORY_USAGE
+		rb_gc_adjust_memory_usage(-this->result_size);
+#endif
+	}
+	this->pgresult = NULL;
+}
+
+static size_t
+pgresult_memsize( t_pg_result *this )
+{
+	return this->result_size;
+}
 
 /*
  * Global functions
@@ -24,23 +120,23 @@ static t_pg_result *pgresult_get_this_safe( VALUE );
 /*
  * Result constructor
  */
-VALUE
-pg_new_result(PGresult *result, VALUE rb_pgconn)
+static VALUE
+pg_new_result2(PGresult *result, VALUE rb_pgconn)
 {
 	int nfields = result ? PQnfields(result) : 0;
 	VALUE self = pgresult_s_allocate( rb_cPGresult );
 	t_pg_result *this;
 
 	this = (t_pg_result *)xmalloc(sizeof(*this) +  sizeof(*this->fnames) * nfields);
-	DATA_PTR(self) = this;
+	RTYPEDDATA_DATA(self) = this;
 
 	this->pgresult = result;
 	this->connection = rb_pgconn;
 	this->typemap = pg_typemap_all_strings;
 	this->p_typemap = DATA_PTR( this->typemap );
-	this->autoclear = 0;
 	this->nfields = -1;
 	this->tuple_hash = Qnil;
+	this->field_map = Qnil;
 
 	PG_ENCODING_SET_NOCHECK(self, ENCODING_GET(rb_pgconn));
 
@@ -58,11 +154,37 @@ pg_new_result(PGresult *result, VALUE rb_pgconn)
 	return self;
 }
 
+VALUE
+pg_new_result(PGresult *result, VALUE rb_pgconn)
+{
+	VALUE self = pg_new_result2(result, rb_pgconn);
+	t_pg_result *this = pgresult_get_this(self);
+	t_pg_connection *p_conn = pg_get_connection(rb_pgconn);
+
+	this->autoclear = 0;
+
+	if( p_conn->guess_result_memsize ){
+		/* Approximate size of underlying pgresult memory storage and account to ruby GC */
+		this->result_size = pgresult_approx_size(result);
+
+#ifdef HAVE_RB_GC_ADJUST_MEMORY_USAGE
+		rb_gc_adjust_memory_usage(this->result_size);
+#endif
+	}
+
+	return self;
+}
+
 VALUE
 pg_new_result_autoclear(PGresult *result, VALUE rb_pgconn)
 {
-	VALUE self = pg_new_result(result, rb_pgconn);
+	VALUE self = pg_new_result2(result, rb_pgconn);
 	t_pg_result *this = pgresult_get_this(self);
+
+	/* Autocleared results are freed implicit instead of by PQclear().
+	 * So it's not very useful to be accounted by ruby GC.
+	 */
+	this->result_size = 0;
 	this->autoclear = 1;
 	return self;
 }
@@ -131,7 +253,13 @@ pg_result_check( VALUE self )
  * call-seq:
  *    res.clear() -> nil
  *
- * Clears the PG::Result object as the result of the query.
+ * Clears the PG::Result object as the result of a query.
+ * This frees all underlying memory consumed by the result object.
+ * Afterwards access to result methods raises PG::Error "result has been cleared".
+ *
+ * Explicit calling #clear can lead to better memory performance, but is not generally necessary.
+ * Special care must be taken when PG::Tuple objects are used.
+ * In this case #clear must not be called unless all PG::Tuple objects of this result are fully materialized.
  *
  * If PG::Result#autoclear? is true then the result is marked as cleared
  * and the underlying C struct will be cleared automatically by libpq.
@@ -141,9 +269,7 @@ VALUE
 pg_result_clear(VALUE self)
 {
 	t_pg_result *this = pgresult_get_this(self);
-	if( !this->autoclear )
-		PQclear(pgresult_get(self));
-	this->pgresult = NULL;
+	pgresult_clear( this );
 	return Qnil;
 }
 
@@ -191,6 +317,7 @@ pgresult_gc_mark( t_pg_result *this )
 	rb_gc_mark( this->connection );
 	rb_gc_mark( this->typemap );
 	rb_gc_mark( this->tuple_hash );
+	rb_gc_mark( this->field_map );
 
 	for( i=0; i < this->nfields; i++ ){
 		rb_gc_mark( this->fnames[i] );
@@ -204,8 +331,7 @@ static void
 pgresult_gc_free( t_pg_result *this )
 {
 	if( !this ) return;
-	if(this->pgresult != NULL && !this->autoclear)
-		PQclear(this->pgresult);
+	pgresult_clear( this );
 
 	xfree(this);
 }
@@ -239,6 +365,20 @@ pgresult_get(VALUE self)
 	return this->pgresult;
 }
 
+
+static const rb_data_type_t pgresult_type = {
+	"pg",
+	{
+		(void (*)(void*))pgresult_gc_mark,
+		(void (*)(void*))pgresult_gc_free,
+		(size_t (*)(const void *))pgresult_memsize,
+	},
+	0, 0,
+#ifdef RUBY_TYPED_FREE_IMMEDIATELY
+	RUBY_TYPED_FREE_IMMEDIATELY,
+#endif
+};
+
 /*
  * Document-method: allocate
  *
@@ -248,7 +388,7 @@ pgresult_get(VALUE self)
 static VALUE
 pgresult_s_allocate( VALUE klass )
 {
-	VALUE self = Data_Wrap_Struct( klass, pgresult_gc_mark, pgresult_gc_free, NULL );
+	VALUE self = TypedData_Wrap_Struct( klass, &pgresult_type, NULL );
 
 	return self;
 }
@@ -279,8 +419,9 @@ static void pgresult_init_fnames(VALUE self)
  *
  * The class to represent the query result tuples (rows).
  * An instance of this class is created as the result of every query.
- * You may need to invoke the #clear method of the instance when finished with
- * the result for better memory performance.
+ *
+ * Since pg-1.1 the amount of memory in use by a PG::Result object is estimated and passed to ruby's garbage collector.
+ * You can invoke the #clear method to force deallocation of memory of the instance when finished with the result for better memory performance.
  *
  * Example:
  *    require 'pg'
@@ -423,7 +564,7 @@ pgresult_ntuples(VALUE self)
 static VALUE
 pgresult_ntuples_for_enum(VALUE self, VALUE args, VALUE eobj)
 {
-    return pgresult_ntuples(self);
+	return pgresult_ntuples(self);
 }
 
 /*
@@ -967,6 +1108,78 @@ pgresult_field_values( VALUE self, VALUE field )
 }
 
 
+/*
+ *  call-seq:
+ *     res.tuple_values( n )   -> array
+ *
+ *  Returns an Array of the field values from the nth row of the result.
+ *
+ */
+static VALUE
+pgresult_tuple_values(VALUE self, VALUE index)
+{
+	int tuple_num = NUM2INT( index );
+	t_pg_result *this;
+	int field;
+	int num_tuples;
+	int num_fields;
+
+	this = pgresult_get_this_safe(self);
+	num_tuples = PQntuples(this->pgresult);
+	num_fields = PQnfields(this->pgresult);
+
+	if ( tuple_num < 0 || tuple_num >= num_tuples )
+		rb_raise( rb_eIndexError, "Index %d is out of range", tuple_num );
+
+	{
+		PG_VARIABLE_LENGTH_ARRAY(VALUE, row_values, num_fields, PG_MAX_COLUMNS)
+
+		/* populate the row */
+		for ( field = 0; field < num_fields; field++ ) {
+			row_values[field] = this->p_typemap->funcs.typecast_result_value(this->p_typemap, self, tuple_num, field);
+		}
+		return rb_ary_new4( num_fields, row_values );
+	}
+}
+
+/*
+ *  call-seq:
+ *     res.tuple( n )   -> PG::Tuple
+ *
+ *  Returns a PG::Tuple from the nth row of the result.
+ *
+ */
+static VALUE
+pgresult_tuple(VALUE self, VALUE index)
+{
+	int tuple_num = NUM2INT( index );
+	t_pg_result *this;
+	int num_tuples;
+
+	this = pgresult_get_this_safe(self);
+	num_tuples = PQntuples(this->pgresult);
+
+	if ( tuple_num < 0 || tuple_num >= num_tuples )
+		rb_raise( rb_eIndexError, "Index %d is out of range", tuple_num );
+
+	if( this->field_map == Qnil ){
+		int i;
+		VALUE field_map = rb_hash_new();
+
+		if( this->nfields == -1 )
+			pgresult_init_fnames( self );
+
+		for( i = 0; i < this->nfields; i++ ){
+			rb_hash_aset(field_map, this->fnames[i], INT2FIX(i));
+		}
+		rb_obj_freeze(field_map);
+		this->field_map = field_map;
+	}
+
+  return pg_tuple_new(self, tuple_num);
+}
+
+
 /*
  * call-seq:
  *    res.each{ |tuple| ... }
@@ -1052,41 +1265,57 @@ pgresult_type_map_get(VALUE self)
 	return this->typemap;
 }
 
-/*
- * call-seq:
- *    res.stream_each{ |tuple| ... }
- *
- * Invokes block for each tuple in the result set in single row mode.
- *
- * This is a convenience method for retrieving all result tuples
- * as they are transferred. It is an alternative to repeated calls of
- * PG::Connection#get_result , but given that it avoids the overhead of
- * wrapping each row into a dedicated result object, it delivers data in nearly
- * the same speed as with ordinary results.
- *
- * The result must be in status PGRES_SINGLE_TUPLE.
- * It iterates over all tuples until the status changes to PGRES_TUPLES_OK.
- * A PG::Error is raised for any errors from the server.
- *
- * Row description data does not change while the iteration. All value retrieval
- * methods refer to only the current row. Result#ntuples returns +1+ while
- * the iteration and +0+ after all tuples were yielded.
- *
- * Example:
- *   conn.send_query( "first SQL query; second SQL query" )
- *   conn.set_single_row_mode
- *   conn.get_result.stream_each do |row|
- *     # do something with the received row of the first query
- *   end
- *   conn.get_result.stream_each do |row|
- *     # do something with the received row of the second query
- *   end
- *   conn.get_result  # => nil   (no more results)
- *
- * Available since PostgreSQL-9.2
- */
+
+static void
+yield_hash(VALUE self, int ntuples, int nfields)
+{
+	int tuple_num;
+	t_pg_result *this = pgresult_get_this(self);
+	UNUSED(nfields);
+
+	for(tuple_num = 0; tuple_num < ntuples; tuple_num++) {
+		rb_yield(pgresult_aref(self, INT2NUM(tuple_num)));
+	}
+
+	pgresult_clear( this );
+}
+
+static void
+yield_array(VALUE self, int ntuples, int nfields)
+{
+	int row;
+	t_pg_result *this = pgresult_get_this(self);
+
+	for ( row = 0; row < ntuples; row++ ) {
+		PG_VARIABLE_LENGTH_ARRAY(VALUE, row_values, nfields, PG_MAX_COLUMNS)
+		int field;
+
+		/* populate the row */
+		for ( field = 0; field < nfields; field++ ) {
+			row_values[field] = this->p_typemap->funcs.typecast_result_value(this->p_typemap, self, row, field);
+		}
+		rb_yield( rb_ary_new4( nfields, row_values ));
+	}
+
+	pgresult_clear( this );
+}
+
+static void
+yield_tuple(VALUE self, int ntuples, int nfields)
+{
+	int tuple_num;
+	t_pg_result *this = pgresult_get_this(self);
+	VALUE result = pg_new_result(this->pgresult, this->connection);
+	UNUSED(nfields);
+
+	for(tuple_num = 0; tuple_num < ntuples; tuple_num++) {
+		VALUE tuple = pgresult_tuple(result, INT2FIX(tuple_num));
+		rb_yield( tuple );
+	}
+}
+
 static VALUE
-pgresult_stream_each(VALUE self)
+pgresult_stream_any(VALUE self, void (*yielder)(VALUE, int, int))
 {
 	t_pg_result *this;
 	int nfields;
@@ -1101,7 +1330,6 @@ pgresult_stream_each(VALUE self)
 	nfields = PQnfields(pgresult);
 
 	for(;;){
-		int tuple_num;
 		int ntuples = PQntuples(pgresult);
 
 		switch( PQresultStatus(pgresult) ){
@@ -1115,14 +1343,7 @@ pgresult_stream_each(VALUE self)
 				pg_result_check( self );
 		}
 
-		for(tuple_num = 0; tuple_num < ntuples; tuple_num++) {
-			rb_yield(pgresult_aref(self, INT2NUM(tuple_num)));
-		}
-
-		if( !this->autoclear ){
-			PQclear( pgresult );
-			this->pgresult = NULL;
-		}
+		yielder( self, ntuples, nfields );
 
 		pgresult = gvl_PQgetResult(pgconn);
 		if( pgresult == NULL )
@@ -1138,6 +1359,46 @@ pgresult_stream_each(VALUE self)
 	return self;
 }
 
+
+/*
+ * call-seq:
+ *    res.stream_each{ |tuple| ... }
+ *
+ * Invokes block for each tuple in the result set in single row mode.
+ *
+ * This is a convenience method for retrieving all result tuples
+ * as they are transferred. It is an alternative to repeated calls of
+ * PG::Connection#get_result , but given that it avoids the overhead of
+ * wrapping each row into a dedicated result object, it delivers data in nearly
+ * the same speed as with ordinary results.
+ *
+ * The base result must be in status PGRES_SINGLE_TUPLE.
+ * It iterates over all tuples until the status changes to PGRES_TUPLES_OK.
+ * A PG::Error is raised for any errors from the server.
+ *
+ * Row description data does not change while the iteration. All value retrieval
+ * methods refer to only the current row. Result#ntuples returns +1+ while
+ * the iteration and +0+ after all tuples were yielded.
+ *
+ * Example:
+ *   conn.send_query( "first SQL query; second SQL query" )
+ *   conn.set_single_row_mode
+ *   conn.get_result.stream_each do |row|
+ *     # do something with each received row of the first query
+ *   end
+ *   conn.get_result.stream_each do |row|
+ *     # do something with each received row of the second query
+ *   end
+ *   conn.get_result  # => nil   (no more results)
+ *
+ * Available since PostgreSQL-9.2
+ */
+static VALUE
+pgresult_stream_each(VALUE self)
+{
+	return pgresult_stream_any(self, yield_hash);
+}
+
 /*
  * call-seq:
  *    res.stream_each_row { |row| ... }
@@ -1153,61 +1414,23 @@ pgresult_stream_each(VALUE self)
 static VALUE
 pgresult_stream_each_row(VALUE self)
 {
-	t_pg_result *this;
-	int row;
-	int nfields;
-	PGconn *pgconn;
-	PGresult *pgresult;
-
-	RETURN_ENUMERATOR(self, 0, NULL);
-
-	this = pgresult_get_this_safe(self);
-	pgconn = pg_get_pgconn(this->connection);
-	pgresult = this->pgresult;
-	nfields = PQnfields(pgresult);
-
-	for(;;){
-		int ntuples = PQntuples(pgresult);
-
-		switch( PQresultStatus(pgresult) ){
-			case PGRES_TUPLES_OK:
-				if( ntuples == 0 )
-					return self;
-				rb_raise( rb_eInvalidResultStatus, "PG::Result is not in single row mode");
-			case PGRES_SINGLE_TUPLE:
-				break;
-			default:
-				pg_result_check( self );
-		}
-
-		for ( row = 0; row < ntuples; row++ ) {
-			PG_VARIABLE_LENGTH_ARRAY(VALUE, row_values, nfields, PG_MAX_COLUMNS)
-			int field;
-
-			/* populate the row */
-			for ( field = 0; field < nfields; field++ ) {
-				row_values[field] = this->p_typemap->funcs.typecast_result_value(this->p_typemap, self, row, field);
-			}
-			rb_yield( rb_ary_new4( nfields, row_values ));
-		}
-
-		if( !this->autoclear ){
-			PQclear( pgresult );
-			this->pgresult = NULL;
-		}
-
-		pgresult = gvl_PQgetResult(pgconn);
-		if( pgresult == NULL )
-			rb_raise( rb_eNoResultError, "no result received - possibly an intersection with another result retrieval");
-
-		if( nfields != PQnfields(pgresult) )
-			rb_raise( rb_eInvalidChangeOfResultFields, "number of fields must not change in single row mode");
-
-		this->pgresult = pgresult;
-	}
+	return pgresult_stream_any(self, yield_array);
+}
 
-	/* never reached */
-	return self;
+/*
+ * call-seq:
+ *    res.stream_each_tuple { |tuple| ... }
+ *
+ * Yields each row of the result set in single row mode.
+ *
+ * This method works equally to #stream_each , but yields a PG::Tuple object.
+ *
+ * Available since PostgreSQL-9.2
+ */
+static VALUE
+pgresult_stream_each_tuple(VALUE self)
+{
+	return pgresult_stream_any(self, yield_tuple);
 }
 
 
@@ -1259,6 +1482,8 @@ init_pg_result()
 	rb_define_method(rb_cPGresult, "values", pgresult_values, 0);
 	rb_define_method(rb_cPGresult, "column_values", pgresult_column_values, 1);
 	rb_define_method(rb_cPGresult, "field_values", pgresult_field_values, 1);
+	rb_define_method(rb_cPGresult, "tuple_values", pgresult_tuple_values, 1);
+	rb_define_method(rb_cPGresult, "tuple", pgresult_tuple, 1);
 	rb_define_method(rb_cPGresult, "cleared?", pgresult_cleared_p, 0);
 	rb_define_method(rb_cPGresult, "autoclear?", pgresult_autoclear_p, 0);
 
@@ -1268,6 +1493,7 @@ init_pg_result()
 	/******     PG::Result INSTANCE METHODS: streaming     ******/
 	rb_define_method(rb_cPGresult, "stream_each", pgresult_stream_each, 0);
 	rb_define_method(rb_cPGresult, "stream_each_row", pgresult_stream_each_row, 0);
+	rb_define_method(rb_cPGresult, "stream_each_tuple", pgresult_stream_each_tuple, 0);
 }