pg_data_encoder 0.1.0 → 0.1.1

data/lib/pg_data_encoder.rb

@@ -1,5 +1,6 @@
 require "pg_data_encoder/version"
 
+require 'pg_data_encoder/temp_buffer'
 require 'pg_data_encoder/encode_for_copy'
 
 module PgDataEncoder

data/lib/pg_data_encoder/encode_for_copy.rb

@@ -3,24 +3,36 @@ require 'stringio'
 module PgDataEncoder
   POSTGRES_EPOCH_TIME = (Time.utc(2000,1,1).to_f * 1_000_000).to_i
 
+  
   class EncodeForCopy
     def initialize(options = {})
       @options = options
       @closed = false
       options[:column_types] ||= {}
       @io = nil
+      @buffer = TempBuffer.new
     end
 
     def add(row)
       setup_io if !@io
       @io.write([row.size].pack("n"))
       row.each_with_index {|col, index|
-        encode_field(@io, col, index)
+        encode_field(@buffer, col, index)
+        if @buffer.size > (1024 * 100)
+          @buffer.rewind
+          @io.write(@buffer.read)
+          @buffer.reopen
+        end
       }
     end
 
     def close
       @closed = true
+      if @buffer.size > 0
+        @buffer.rewind
+        @io.write(@buffer.read)
+        @buffer.reopen
+      end
       @io.write([-1].pack("n")) rescue raise Exception.new("No rows have been added to the encoder!")
       @io.rewind
     end
@@ -86,7 +98,7 @@ module PgDataEncoder
           io.write(buf)
         end
       when Array
-        array_io = StringIO.new
+        array_io = TempBuffer.new
         field.compact!
         completed = false
         case field[0]
@@ -147,7 +159,7 @@ module PgDataEncoder
         end
       when Hash
         raise Exception.new("Hash's can't contain hashes") if depth > 0
-        hash_io = StringIO.new
+        hash_io = TempBuffer.new
         
         hash_io.write([field.size].pack("N"))
         field.each_pair {|key,val|

data/lib/pg_data_encoder/temp_buffer.rb

@@ -0,0 +1,27 @@
+class TempBuffer
+  def initialize
+    @st = ""
+  end
+  def size 
+    @st.size
+  end
+
+  def write(st)
+    @st << st
+  end
+  def rewind
+
+  end
+  def reopen
+    @st = ""
+  end
+  def read
+    @st
+  end
+  def pos
+    @st.size
+  end
+  def string
+    @st
+  end
+end

data/lib/pg_data_encoder/version.rb

@@ -1,3 +1,3 @@
 module PgDataEncoder
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/spec/big_write_spec.rb

@@ -0,0 +1,15 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require 'benchmark'
+describe "testing changes with large imports and speed issues" do
+  it 'should import lots of data quickly' do
+    encoder = PgDataEncoder::EncodeForCopy.new(temp_file: true)
+    puts Benchmark.measure {
+        0.upto(100_000) {|r|
+            encoder.add [1, "text", {a: 1, b: "asdf"}]
+        }
+    }
+    encoder.close
+    io = encoder.get_io
+
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pg_data_encoder
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-19 00:00:00.000000000 Z
+date: 2014-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -59,8 +59,10 @@ files:
 - examples/fast_load.rb
 - lib/pg_data_encoder.rb
 - lib/pg_data_encoder/encode_for_copy.rb
+- lib/pg_data_encoder/temp_buffer.rb
 - lib/pg_data_encoder/version.rb
 - pg_data_encoder.gemspec
+- spec/big_write_spec.rb
 - spec/errors_spec.rb
 - spec/fixtures/3_col_array.txt
 - spec/fixtures/3_col_hstore.dat
@@ -111,6 +113,7 @@ specification_version: 3
 summary: for faster input of data into postgres you can use this to generate the binary
   import and run COPY FROM
 test_files:
+- spec/big_write_spec.rb
 - spec/errors_spec.rb
 - spec/fixtures/3_col_array.txt
 - spec/fixtures/3_col_hstore.dat