pg_data_encoder 0.0.1 → 0.0.2

data/README.md

@@ -14,10 +14,9 @@ With it you can make a bulk insert like this
 
     Product.pg_copy_from(encoder.get_io, :format => :binary, :columns => [:id, :name, :desc])
 
-With a fairly complicated table that includes an index and an hstore + index.
-I can get 1600 inserts/sec on my overworked macbook pro.
+## Try it out yourself,   in the examples folder there is a simple test
 
-Your usage may vary
+on my i3 box with an ssd drive I can get 270,000 inserts a second with an hstore and indexes
 
 NOTE: Only a few of the many data types are supported.  check below for more details
 
@@ -62,12 +61,10 @@ or
 
 
 
-
 ## Added type support
 
-  Currently it supports Integers, Strings, Hstore.
+  Currently it supports Integers, Strings, Hstore, Floats (double precision), Timestamp.
 
-  Help would be appreciated for DateTime, Float, Double, ...
 ## Contributing
 
 

data/examples/fast_load.rb

@@ -0,0 +1,47 @@
+require 'active_record'
+require 'postgres-copy'
+require 'pg_data_encoder'
+require 'benchmark'
+# Create a test db before running
+# add any needed username, password, port
+# install the required gems
+#
+# gem install postgres-copy pg_data_encoder activerecord --no-ri --no-rdoc
+
+
+ActiveSupport.on_load :active_record do
+  require "postgres-copy/active_record"
+end
+ActiveRecord::Base.establish_connection(
+        :adapter  => "postgresql",
+        :host     => "localhost",
+        :database => "test"
+)
+ActiveRecord::Base.connection.execute %{
+  SET client_min_messages TO warning;
+  DROP TABLE IF EXISTS test_models;
+  CREATE TABLE test_models (id serial PRIMARY KEY, data VARCHAR);
+}
+
+class TestModel < ActiveRecord::Base
+end
+
+encoder = PgDataEncoder::EncodeForCopy.new
+
+puts "Loading data to disk"
+puts Benchmark.measure {
+  0.upto(1_000_000).each {|i|
+    encoder.add ["test data"]
+  }
+}
+puts "inserting into db"
+puts Benchmark.measure {
+  TestModel.pg_copy_from(encoder.get_io, :format => :binary, :columns => [:data])
+}
+
+encoder.remove
+# Results on my i5 with ssd backed postgres server
+# 11.7 seconds to generate data file.   3.7 seconds to insert 1,000,000 simple items into a table.
+#
+# 11.670000   0.010000  11.680000 ( 11.733414)
+#  0.030000   0.000000   0.030000 (  3.782371)

data/lib/pg_data_encoder/encode_for_copy.rb

@@ -1,6 +1,7 @@
 require 'tempfile'
 require 'stringio'
 module PgDataEncoder
+  POSTGRES_EPOCH_DATE = (Time.utc(2000,1,1).to_f * 1_000_000).to_i
   class EncodeForCopy
     def initialize(options = {})
       @options = options
@@ -10,7 +11,6 @@ module PgDataEncoder
 
     def add(row)
       setup_io if !@io
-
       @io.write([row.size].pack("n"))
       row.each {|col|
         encode_field(@io, col)
@@ -56,6 +56,10 @@ module PgDataEncoder
         buf = [field].pack("N")
         io.write([buf.bytesize].pack("N"))
         io.write(buf)
+      when Float
+        buf = [field].pack("G")
+        io.write([buf.bytesize].pack("N"))
+        io.write(buf)
       when nil
         io.write([-1].pack("N"))
       when String
@@ -75,6 +79,10 @@ module PgDataEncoder
         }
         io.write([hash_io.pos].pack("N"))  # assumed identifier for hstore column
         io.write(hash_io.string)
+      when Time
+        buf = [(field.to_f * 1_000_000 - POSTGRES_EPOCH_DATE).to_i].pack("L!>")
+        io.write([buf.bytesize].pack("N"))
+        io.write(buf)
       else
         raise Exception.new("Unsupported Format: #{field.class.name}")
       end

data/lib/pg_data_encoder/version.rb

@@ -1,3 +1,3 @@
 module PgDataEncoder
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/verify_data_formats_spec.rb

@@ -27,4 +27,57 @@ describe "generating data" do
     str.should == existing_data
   end
 
+
+  it 'should encode timestamp data correctly' do
+    encoder = PgDataEncoder::EncodeForCopy.new
+    encoder.add [Time.parse("2013-06-11 15:03:54.62605 UTC")]
+    encoder.close
+    io = encoder.get_io
+    existing_data = filedata("timestamp.dat")
+    str = io.read
+    io.class.name.should == "StringIO"
+    str.force_encoding("ASCII-8BIT")
+    #File.open("spec/fixtures/output.dat", "w:ASCII-8BIT") {|out| out.write(str) }
+    str.should == existing_data
+  end
+
+  it 'should encode float correctly from tempfile' do
+    encoder = PgDataEncoder::EncodeForCopy.new(:use_tempfile => true)
+    encoder.add [Time.parse("2013-06-11 15:03:54.62605 UTC")]
+    encoder.close
+    io = encoder.get_io
+    existing_data = filedata("timestamp.dat")
+    str = io.read
+    io.class.name.should == "Tempfile"
+    str.force_encoding("ASCII-8BIT")
+    #File.open("spec/fixtures/output.dat", "w:ASCII-8BIT") {|out| out.write(str) }
+    str.should == existing_data
+  end
+
+  it 'should encode float data correctly' do
+    encoder = PgDataEncoder::EncodeForCopy.new
+    encoder.add [1234567.1234567]
+    encoder.close
+    io = encoder.get_io
+    existing_data = filedata("float.dat")
+    str = io.read
+    io.class.name.should == "StringIO"
+    str.force_encoding("ASCII-8BIT")
+    #File.open("spec/fixtures/output.dat", "w:ASCII-8BIT") {|out| out.write(str) }
+    str.should == existing_data
+  end
+
+  it 'should encode float correctly from tempfile' do
+    encoder = PgDataEncoder::EncodeForCopy.new(:use_tempfile => true)
+    encoder.add [1234567.1234567]
+    encoder.close
+    io = encoder.get_io
+    existing_data = filedata("float.dat")
+    str = io.read
+    io.class.name.should == "Tempfile"
+    str.force_encoding("ASCII-8BIT")
+    #File.open("spec/fixtures/output.dat", "w:ASCII-8BIT") {|out| out.write(str) }
+    str.should == existing_data
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pg_data_encoder
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-12 00:00:00.000000000 Z
+date: 2013-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -56,13 +56,16 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/fast_load.rb
 - lib/pg_data_encoder.rb
 - lib/pg_data_encoder/encode_for_copy.rb
 - lib/pg_data_encoder/version.rb
 - pg_data_encoder.gemspec
 - spec/fixtures/3_col_hstore.dat
 - spec/fixtures/3_col_hstore.txt
+- spec/fixtures/float.dat
 - spec/fixtures/output.dat
+- spec/fixtures/timestamp.dat
 - spec/spec_helper.rb
 - spec/verify_data_formats_spec.rb
 homepage: https://github.com/pbrumm/pg_data_encoder
@@ -85,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: for faster input of data into postgres you can use this to generate the binary
@@ -93,6 +96,8 @@ summary: for faster input of data into postgres you can use this to generate the
 test_files:
 - spec/fixtures/3_col_hstore.dat
 - spec/fixtures/3_col_hstore.txt
+- spec/fixtures/float.dat
 - spec/fixtures/output.dat
+- spec/fixtures/timestamp.dat
 - spec/spec_helper.rb
 - spec/verify_data_formats_spec.rb