pg 0.17.1-x86-mingw32 → 0.18.0.pre20141017160319-x86-mingw32

data/History.rdoc

@@ -1,3 +1,9 @@
+== v0.18.0 [unreleased]
+
+Enhancements:
+
+- Add an extensible type cast system.
+
 == v0.17.1 [2013-12-18] Michael Granger <ged@FaerieMUD.org>
 
 Bugfixes:

data/Manifest.txt

@@ -20,17 +20,35 @@ ext/gvl_wrappers.c
 ext/gvl_wrappers.h
 ext/pg.c
 ext/pg.h
+ext/pg_binary_decoder.c
+ext/pg_binary_encoder.c
+ext/pg_coder.c
 ext/pg_connection.c
+ext/pg_copy_coder.c
 ext/pg_errors.c
 ext/pg_result.c
+ext/pg_text_decoder.c
+ext/pg_text_encoder.c
+ext/pg_type_map_all_strings.c
+ext/pg_type_map_by_column.c
+ext/pg_type_map_by_mri_type.c
+ext/pg_type_map_by_oid.c
+ext/pg_type_map.c
+ext/util.c
+ext/util.h
 ext/vc/pg.sln
 ext/vc/pg_18/pg.vcproj
 ext/vc/pg_19/pg_19.vcproj
 lib/pg.rb
+lib/pg/basic_type_mapping.rb
+lib/pg/coder.rb
 lib/pg/connection.rb
 lib/pg/constants.rb
 lib/pg/exceptions.rb
 lib/pg/result.rb
+lib/pg/text_decoder.rb
+lib/pg/text_encoder.rb
+lib/pg/type_map_by_column.rb
 sample/array_insert.rb
 sample/async_api.rb
 sample/async_copyto.rb
@@ -51,7 +69,13 @@ sample/wal_shipper.rb
 sample/warehouse_partitions.rb
 spec/data/expected_trace.out
 spec/data/random_binary_data
-spec/lib/helpers.rb
+spec/helpers.rb
+spec/pg/basic_type_mapping_spec.rb
 spec/pg/connection_spec.rb
 spec/pg/result_spec.rb
+spec/pg/type_map_by_column_spec.rb
+spec/pg/type_map_by_mri_type_spec.rb
+spec/pg/type_map_by_oid_spec.rb
+spec/pg/type_map_spec.rb
+spec/pg/type_spec.rb
 spec/pg_spec.rb

data/README.rdoc

@@ -64,6 +64,53 @@ There's also {a Google+ group}[http://goo.gl/TFy1U] and a
 want to chat about something.
 
 
+== Type Casts
+
+Pg can optionally type cast result values and query parameters in Ruby or
+native C code. This can speed up data transfers to and from the database,
+because String allocations are reduced and conversions in (slower) Ruby code
+can be omitted.
+
+Very basic type casting can be enabled by:
+
+    conn.type_map_for_results = PG::BasicTypeMapForResults.new conn
+    # ... this works for result value mapping:
+    conn.exec("select 1, now(), '{2,3}'::int[]").values
+        # => [[1, 2014-09-21 20:51:56 +0200, [2, 3]]]
+
+    conn.type_map_for_queries = PG::BasicTypeMapForQueries.new conn
+    # ... and this for param value mapping:
+    conn.exec_params("SELECT $1::text, $2::text, $3::text", [1, 1.23, [2,3]]).values
+        # => [["1", "1.2300000000000000E+00", "{2,3}"]]
+
+But Pg's type casting is highly customizable. That's why it's divided into
+2 layers:
+
+=== Encoders / Decoders (ext/pg_*coder.c, lib/pg/*coder.rb)
+
+This is the lower layer, containing encoding classes that convert Ruby
+objects for transmission to the DBMS and decoding classes to convert
+received data back to Ruby objects. The classes are namespaced according
+to their format and direction in PG::TextEncoder, PG::TextDecoder,
+PG::BinaryEncoder and PG::BinaryDecoder.
+
+It is possible to assign a type OID, format code (text or binary) and
+optionally a name to an encoder or decoder object. It's also possible
+to build composite types by assigning an element encoder/decoder.
+PG::Coder objects can be used to set up a PG::TypeMap or alternatively
+to convert single values to/from their string representation.
+
+=== PG::TypeMap and derivations (ext/pg_type_map*.c, lib/pg/type_map*.rb)
+
+A TypeMap defines which value will be converted by which encoder/decoder.
+There are different type map strategies, implemented by several derivations
+of this class. They can be chosen and configured according to the particular
+needs for type casting.
+
+A type map can be assigned per connection or per query respectively per
+result set. Type maps can also be used for COPY in and out data streaming.
+
+
 == Contributing
 
 To report bugs, suggest features, or check out the source with Mercurial,

data/Rakefile

@@ -53,14 +53,16 @@ $hoespec = Hoe.spec 'pg' do
 	self.extra_rdoc_files = Rake::FileList[ '*.rdoc' ]
 	self.extra_rdoc_files.include( 'POSTGRES', 'LICENSE' )
 	self.extra_rdoc_files.include( 'ext/*.c' )
+	self.license :BSD
 
 	self.developer 'Michael Granger', 'ged@FaerieMUD.org'
 	self.developer 'Lars Kanis', 'lars@greiz-reinsdorf.de'
 
 	self.dependency 'rake-compiler', '~> 0.9', :developer
-	self.dependency 'hoe', '~> 3.5.1', :developer
-	self.dependency 'hoe-deveiate', '~> 0.2', :developer
+	self.dependency 'hoe', '~> 3.12', :developer
+	self.dependency 'hoe-deveiate', '~> 0.6', :developer
 	self.dependency 'hoe-bundler', '~> 1.0', :developer
+	self.dependency 'rspec', '~> 3.0', :developer
 
 	self.spec_extras[:licenses] = ['BSD', 'Ruby', 'GPL']
 	self.spec_extras[:extensions] = [ 'ext/extconf.rb' ]
@@ -118,16 +120,23 @@ Rake::ExtensionTask.new do |ext|
 	ext.cross_compile  = true
 	ext.cross_platform = CrossLibraries.map &:for_platform
 
-  ext.cross_config_options += CrossLibraries.map do |lib|
-    {
-      lib.for_platform => [
-        "--with-pg-include=#{lib.static_postgresql_libdir}",
-        "--with-opt-include=#{lib.static_postgresql_incdir}",
-        "--with-pg-lib=#{lib.static_postgresql_libdir}",
-        "--with-opt-lib=#{lib.static_openssl_builddir}",
-      ]
-    }
-  end
+	ext.cross_config_options += CrossLibraries.map do |lib|
+		{
+			lib.for_platform => [
+				"--enable-windows-cross",
+				"--with-pg-include=#{lib.static_postgresql_incdir}",
+				"--with-pg-lib=#{lib.static_postgresql_libdir}",
+				# libpq-fe.h resides in src/interfaces/libpq/ before make install
+				"--with-opt-include=#{lib.static_postgresql_libdir}",
+			]
+		}
+	end
+
+	# Add libpq.dll to windows binary gemspec
+	ext.cross_compiling do |spec|
+		# mingw32-platform strings differ (RUBY_PLATFORM=i386-mingw32 vs. x86-mingw32 for rubygems)
+		spec.files << "lib/#{spec.platform.to_s.gsub(/^x86-/, "i386-")}/libpq.dll"
+	end
 end
 
 

data/Rakefile.cross

@@ -28,8 +28,8 @@ class CrossLibrary < OpenStruct
 		self.openssl_config             = openssl_config
 
 		# Cross-compilation constants
-		self.openssl_version            = ENV['OPENSSL_VERSION'] || '1.0.1e'
-		self.postgresql_version         = ENV['POSTGRESQL_VERSION'] || '9.2.3'
+		self.openssl_version            = ENV['OPENSSL_VERSION'] || '1.0.1i'
+		self.postgresql_version         = ENV['POSTGRESQL_VERSION'] || '9.3.5'
 
 		self.compile_home               = Pathname( "./build" ).expand_path
 		self.static_sourcesdir          = compile_home + 'sources'
@@ -64,7 +64,7 @@ class CrossLibrary < OpenStruct
 		self.postgresql_global_makefile = static_postgresql_srcdir + 'Makefile.global'
 		self.postgresql_shlib_makefile  = static_postgresql_srcdir + 'Makefile.shlib'
 		self.postgresql_shlib_mf_orig   = static_postgresql_srcdir + 'Makefile.shlib.orig'
-		self.postgresql_lib             = static_postgresql_libdir + 'libpq.a'
+		self.postgresql_lib             = static_postgresql_libdir + 'libpq.dll'
 		self.postgresql_patches         = Rake::FileList[ (MISCDIR + "postgresql-#{postgresql_version}.*.patch").to_s ]
 
 		# Use rake-compilers config.yml to determine the toolchain that was used
@@ -82,7 +82,7 @@ class CrossLibrary < OpenStruct
 		CLEAN.include( static_builddir.to_s )
 
 
-		ENV['RUBY_CC_VERSION'] ||= '1.8.7:1.9.3:2.0.0'
+		ENV['RUBY_CC_VERSION'] ||= '1.9.3:2.0.0'
 
 		def download(url, save_to)
 			part = save_to+".part"
@@ -136,7 +136,7 @@ class CrossLibrary < OpenStruct
 
 		# generate the makefile in a clean build location
 		file openssl_makefile => static_openssl_builddir do |t|
-			Dir.chdir( static_openssl_builddir ) do
+			chdir( static_openssl_builddir ) do
 				cmd = cmd_prelude.dup
 				cmd << "./Configure" << openssl_config
 
@@ -148,7 +148,7 @@ class CrossLibrary < OpenStruct
 		task :openssl_libs => [ libssleay32, libeay32 ]
 
 		task :compile_static_openssl => openssl_makefile do |t|
-			Dir.chdir( static_openssl_builddir ) do
+			chdir( static_openssl_builddir ) do
 				cmd = cmd_prelude.dup
 				cmd << 'make' << "-j#{NUM_CPUS}" << 'build_libs'
 
@@ -184,7 +184,6 @@ class CrossLibrary < OpenStruct
 			puts "extracting %s to %s" % [ postgresql_tarball, static_postgresql_builddir.parent ]
 			static_postgresql_builddir.mkpath
 			run 'tar', '-xjf', postgresql_tarball.to_s, '-C', static_postgresql_builddir.parent.to_s
-			mv postgresql_shlib_makefile, postgresql_shlib_mf_orig
 
 			postgresql_patches.each do |patchfile|
 				puts "  applying patch #{patchfile}..."
@@ -200,17 +199,15 @@ class CrossLibrary < OpenStruct
 				"--host=#{host_platform}",
 				'--with-openssl',
 				'--without-zlib',
-				'--disable-shared',
 			]
 
-			Dir.chdir( static_postgresql_builddir ) do
+			chdir( static_postgresql_builddir ) do
 				configure_path = static_postgresql_builddir + 'configure'
 				cmd = [ configure_path.to_s, *options ]
 				cmd << "CFLAGS=-L#{static_openssl_builddir}"
 				cmd << "LDFLAGS=-L#{static_openssl_builddir}"
 				cmd << "LDFLAGS_SL=-L#{static_openssl_builddir}"
 				cmd << "LIBS=-lwsock32 -lgdi32"
-                                cmd << "DLLWRAP_FLAGS=-lcrypt32"
 				cmd << "CPPFLAGS=-I#{static_openssl_builddir}/include"
 
 				run( *cmd )
@@ -218,31 +215,32 @@ class CrossLibrary < OpenStruct
 		end
 
 
-		# patch the Makefile.shlib -- depend on the build dir so it's only
-		# rewritten if the tarball is re-extracted.
-		file postgresql_shlib_makefile => postgresql_shlib_mf_orig do |t|
-			tf = Tempfile.new( postgresql_shlib_makefile.basename.to_s )
-			postgresql_shlib_mf_orig.open( File::RDONLY ) do |ifh|
-				ifh.each_line do |line|
-					tf.print( line.sub(/^(\s*haslibarule\s*=\s*yes)/, "# \\1 ") )
-				end
+		# make libpq.dll
+		task postgresql_lib => [ postgresql_global_makefile ] do |t|
+			chdir( postgresql_lib.dirname ) do
+				sh 'make',
+					"-j#{NUM_CPUS}",
+					postgresql_lib.basename.to_s,
+					'SHLIB_LINK=-lssleay32 -leay32 -lcrypt32 -lgdi32 -lsecur32 -lwsock32 -lws2_32'
 			end
-			tf.close
-
-			FileUtils.mv( tf.path, t.name, :verbose => $puts )
 		end
 
 
-		# make libpq.a
-		task postgresql_lib => [ postgresql_global_makefile, postgresql_shlib_makefile ] do |t|
-			Dir.chdir( postgresql_lib.dirname ) do
-				sh 'make', "-j#{NUM_CPUS}", postgresql_lib.basename.to_s, 'PORTNAME=win32'
-			end
-		end
+		#desc 'compile libpg.a'
+		task :libpq => postgresql_lib
 
+		# copy libpq.dll to lib dir
+		dest_libpq = "lib/#{for_platform}/#{postgresql_lib.basename}"
+		directory File.dirname(dest_libpq)
+		file dest_libpq => [postgresql_lib, File.dirname(dest_libpq)] do
+			cp postgresql_lib, dest_libpq
+		end
 
-		#desc 'compile static libpg.a'
-		task :static_libpq => postgresql_lib
+		stage_libpq = "tmp/#{for_platform}/stage/#{dest_libpq}"
+		directory File.dirname(stage_libpq)
+		file stage_libpq => [postgresql_lib, File.dirname(stage_libpq)] do |t|
+			cp postgresql_lib, stage_libpq
+		end
 	end
 end
 
@@ -259,10 +257,7 @@ else
 end
 
 desc 'cross compile pg for win32'
-task :cross do
-	ENV['CROSS_COMPILING'] = 'yes'
-end
-task :cross => [ :mingw32, :static_libpq ]
+task :cross => [ :mingw32, :libpq ]
 
 task :mingw32 do
 	# Use Rake::ExtensionCompiler helpers to find the proper host
@@ -272,3 +267,14 @@ task :mingw32 do
 		fail
 	end
 end
+
+# To reduce the gem file size strip mingw32 dlls before packaging
+ENV['RUBY_CC_VERSION'].to_s.split(':').each do |ruby_version|
+  task "tmp/i386-mingw32/stage/lib/#{ruby_version[/^\d+\.\d+/]}/pg_ext.so" do |t|
+    sh "i686-w64-mingw32-strip -S tmp/i386-mingw32/stage/lib/#{ruby_version[/^\d+\.\d+/]}/pg_ext.so"
+  end
+
+  task "tmp/x64-mingw32/stage/lib/#{ruby_version[/^\d+\.\d+/]}/pg_ext.so" do |t|
+    sh "x86_64-w64-mingw32-strip -S tmp/x64-mingw32/stage/lib/#{ruby_version[/^\d+\.\d+/]}/pg_ext.so"
+  end
+end

data/ext/extconf.rb

@@ -15,36 +15,35 @@ if pgdir = with_config( 'pg' )
 	ENV['PATH'] = "#{pgdir}/bin" + File::PATH_SEPARATOR + ENV['PATH']
 end
 
-if ENV['CROSS_COMPILING']
-	$LDFLAGS << " -L#{CONFIG['libdir']}"
-
-	# Link against all required libraries for static build, if they are available
-	have_library( 'crypt32', 'CertOpenStore' ) && append_library( $libs, 'crypt32' )
-	have_library( 'gdi32', 'CreateDC' ) && append_library( $libs, 'gdi32' )
-	have_library( 'secur32' ) && append_library( $libs, 'secur32' )
-	have_library( 'ws2_32', 'WSASocket') && append_library( $libs, 'ws2_32' )
-	have_library( 'crypto', 'BIO_new' ) && append_library( $libs, 'crypto' )
-	have_library( 'ssl', 'SSL_new' ) && append_library( $libs, 'ssl' )
-end
+if enable_config("windows-cross")
+	# Avoid dependency to external libgcc.dll on x86-mingw32
+	$LDFLAGS << " -static-libgcc"
+	# Don't use pg_config for cross build, but --with-pg-* path options
+	dir_config 'pg'
 
-if pgconfig = ( with_config('pg-config') || with_config('pg_config') || find_executable('pg_config') )
-	$stderr.puts "Using config values from %s" % [ pgconfig ]
-	incdir = `"#{pgconfig}" --includedir`.chomp
-	libdir = `"#{pgconfig}" --libdir`.chomp
-	dir_config 'pg', incdir, libdir
-
-	# Try to use runtime path linker option, even if RbConfig doesn't know about it.
-	# The rpath option is usually set implicit by dir_config(), but so far not
-	# on MacOS-X.
-	if RbConfig::CONFIG["RPATHFLAG"].to_s.empty? && try_link('int main() {return 0;}', " -Wl,-rpath,#{libdir}")
-		$LDFLAGS << " -Wl,-rpath,#{libdir}"
-	end
 else
-	$stderr.puts "No pg_config... trying anyway. If building fails, please try again with",
-		" --with-pg-config=/path/to/pg_config"
-	dir_config 'pg'
+	# Native build
+
+	if pgconfig = ( with_config('pg-config') || with_config('pg_config') || find_executable('pg_config') )
+		$stderr.puts "Using config values from %s" % [ pgconfig ]
+		incdir = `"#{pgconfig}" --includedir`.chomp
+		libdir = `"#{pgconfig}" --libdir`.chomp
+		dir_config 'pg', incdir, libdir
+
+		# Try to use runtime path linker option, even if RbConfig doesn't know about it.
+		# The rpath option is usually set implicit by dir_config(), but so far not
+		# on MacOS-X.
+		if RbConfig::CONFIG["RPATHFLAG"].to_s.empty? && try_link('int main() {return 0;}', " -Wl,-rpath,#{libdir}")
+			$LDFLAGS << " -Wl,-rpath,#{libdir}"
+		end
+	else
+		$stderr.puts "No pg_config... trying anyway. If building fails, please try again with",
+			" --with-pg-config=/path/to/pg_config"
+		dir_config 'pg'
+	end
 end
 
+
 find_header( 'libpq-fe.h' ) or abort "Can't find the 'libpq-fe.h header"
 find_header( 'libpq/libpq-fs.h' ) or abort "Can't find the 'libpq/libpq-fs.h header"
 find_header( 'pg_config_manual.h' ) or abort "Can't find the 'pg_config_manual.h' header"
@@ -73,6 +72,7 @@ have_func 'PQsetClientEncoding'
 have_func 'PQlibVersion'
 have_func 'PQping'
 have_func 'PQsetSingleRowMode'
+have_func 'PQconninfo'
 
 have_func 'rb_encdb_alias'
 have_func 'rb_enc_alias'
@@ -80,6 +80,7 @@ have_func 'rb_thread_call_without_gvl'
 have_func 'rb_thread_call_with_gvl'
 have_func 'rb_thread_fd_select'
 have_func 'rb_w32_wrap_io_handle'
+have_func 'rb_str_modify_expand'
 
 have_const 'PGRES_COPY_BOTH', 'libpq-fe.h'
 have_const 'PGRES_SINGLE_TUPLE', 'libpq-fe.h'

data/ext/pg.c

@@ -123,24 +123,6 @@ const char * const (pg_enc_pg2ruby_mapping[][2]) = {
  * A cache of mapping from PostgreSQL's encoding indices to Ruby's rb_encoding*s.
  */
 static struct st_table *enc_pg2ruby;
-static ID s_id_index;
-
-
-/*
- * Get the index of encoding +val+.
- * :FIXME: Look into replacing this with rb_enc_get_index() since 1.9.1 isn't really
- * used anymore.
- */
-int
-pg_enc_get_index(VALUE val)
-{
-	int i = ENCODING_GET_INLINED(val);
-	if (i == ENCODING_INLINE_MAX) {
-		VALUE iv = rb_ivar_get(val, s_id_index);
-		i = NUM2INT(iv);
-	}
-	return i;
-}
 
 
 /*
@@ -249,6 +231,68 @@ pg_get_rb_encoding_as_pg_encoding( rb_encoding *enc )
 #endif /* M17N_SUPPORTED */
 
 
+/*
+ * Ensures that the given string has enough capacity to take expand_len
+ * more data bytes. The new data part of the String is not initialized.
+ *
+ * current_out must be a pointer within the data part of the String object.
+ * This pointer is returned and possibly adjusted, because the location of the data
+ * part of the String can change through this function.
+ *
+ * PG_RB_STR_ENSURE_CAPA can be used to do fast inline checks of the remaining capacity.
+ * end_capa it is then set to the first byte after the currently reserved memory,
+ * if not NULL.
+ *
+ * Before the String can be used with other string functions or returned to Ruby space,
+ * the string length has to be set with rb_str_set_len().
+ *
+ * Usage example:
+ *
+ *   VALUE string;
+ *   char *current_out, *end_capa;
+ *   PG_RB_STR_NEW( string, current_out, end_capa );
+ *   while( data_is_going_to_be_processed ){
+ *     PG_RB_STR_ENSURE_CAPA( string, 2 current_out, end_capa );
+ *     *current_out++ = databyte1;
+ *     *current_out++ = databyte2;
+ *   }
+ *   rb_str_set_len( string, current_out - RSTRING_PTR(string) );
+ *
+ */
+#ifdef HAVE_RB_STR_MODIFY_EXPAND
+	/* Use somewhat faster version with access to string capacity on MRI */
+	char *
+	pg_rb_str_ensure_capa( VALUE str, long expand_len, char *curr_ptr, char **end_ptr )
+	{
+		long curr_len = curr_ptr - RSTRING_PTR(str);
+		long curr_capa = rb_str_capacity( str );
+		if( curr_capa < curr_len + expand_len ){
+			rb_str_set_len( str, curr_len );
+			rb_str_modify_expand( str, (curr_len + expand_len) * 2 - curr_capa );
+			curr_ptr = RSTRING_PTR(str) + curr_len;
+		}
+		if( end_ptr )
+			*end_ptr = RSTRING_PTR(str) + rb_str_capacity( str );
+		return curr_ptr;
+	}
+#else
+	/* Use the more portable version */
+	char *
+	pg_rb_str_ensure_capa( VALUE str, long expand_len, char *curr_ptr, char **end_ptr )
+	{
+		long curr_len = curr_ptr - RSTRING_PTR(str);
+		long curr_capa = RSTRING_LEN( str );
+		if( curr_capa < curr_len + expand_len ){
+			rb_str_resize( str, (curr_len + expand_len) * 2 - curr_capa );
+			curr_ptr = RSTRING_PTR(str) + curr_len;
+		}
+		if( end_ptr )
+			*end_ptr = RSTRING_PTR(str) + RSTRING_LEN(str);
+		return curr_ptr;
+	}
+#endif
+
+
 /**************************************************************************
  * Module Methods
  **************************************************************************/
@@ -534,12 +578,22 @@ Init_pg_ext()
 
 #ifdef M17N_SUPPORTED
 	enc_pg2ruby = st_init_numtable();
-	s_id_index = rb_intern("@encoding");
 #endif
 
 	/* Initialize the main extension classes */
 	init_pg_connection();
 	init_pg_result();
 	init_pg_errors();
+	init_pg_type_map();
+	init_pg_type_map_all_strings();
+	init_pg_type_map_by_column();
+	init_pg_type_map_by_mri_type();
+	init_pg_type_map_by_oid();
+	init_pg_coder();
+	init_pg_text_encoder();
+	init_pg_text_decoder();
+	init_pg_binary_encoder();
+	init_pg_binary_decoder();
+	init_pg_copycoder();
 }