pg_data_encoder 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pg_data_encoder.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Pete Brumm
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,79 @@
+# PgDataEncoder
+
+Creates a binary data file that can be imported into postgres's copy from command
+
+Works well in collaboration with the postgres-copy gem
+
+    https://github.com/diogob/postgres-copy
+
+With it you can make a bulk insert like this
+
+    encoder = PgDataEncoder::EncodeForCopy.new
+    encoder.add [1, "test", "first"]
+    encoder.add [2, "test2", "second"]
+
+    Product.pg_copy_from(encoder.get_io, :format => :binary, :columns => [:id, :name, :desc])
+
+With a fairly complicated table that includes an index and an hstore + index.
+I can get 1600 inserts/sec on my overworked macbook pro.
+
+Your usage may vary
+
+NOTE: Only a few of the many data types are supported.  check below for more details
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'pg_data_encoder'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pg_data_encoder
+
+## Usage
+
+    pg = PgDataEncoder::EncodeForCopy.new
+    pg.add([1,2,3,4,"text"])
+    io = pg.get_io
+
+For large imports you can use the use_tempfile => true option to enable Tempfile usage.   otherwise it uses StringIO
+
+    pg = PgDataEncoder::EncodeForCopy.new(use_tempfile: true)
+    pg.add([1,2,3,4,"text"])
+    io = pg.get_io
+
+    pg.remove  # to delete your file
+
+## Notes
+
+
+Columns must line up on the incoming table.   if they don't you need to filter the copy to not need them
+
+    COPY table_name FROM STDIN BINARY
+
+or
+
+    COPY table_name(field1, field2) FROM STDIN BINARY
+
+
+
+
+## Added type support
+
+  Currently it supports Integers, Strings, Hstore.
+
+  Help would be appreciated for DateTime, Float, Double, ...
+## Contributing
+
+
+
+1. Fork it
+2. Create your feature branch (`git checkout -b feature/new_feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin feature/new_feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/pg_data_encoder/encode_for_copy.rb

@@ -0,0 +1,84 @@
+require 'tempfile'
+require 'stringio'
+module PgDataEncoder
+  class EncodeForCopy
+    def initialize(options = {})
+      @options = options
+      @closed = false
+      @io = nil
+    end
+
+    def add(row)
+      setup_io if !@io
+
+      @io.write([row.size].pack("n"))
+      row.each {|col|
+        encode_field(@io, col)
+      }
+    end
+
+    def close
+      @closed = true
+      @io.write([-1].pack("n"))
+      @io.rewind
+    end
+
+    def get_io
+      if !@closed
+        close
+      end
+      @io
+    end
+
+    def remove
+      if @io.kind_of?(Tempfile)
+        @io.close
+        @io.unlink
+      end
+    end
+
+    private
+
+    def setup_io
+      if @options[:use_tempfile] == true
+        @io = Tempfile.new("copy_binary", :encoding => 'ascii-8bit')
+        @io.unlink
+      else
+        @io = StringIO.new
+      end
+      @io.write("PGCOPY\n\377\r\n\0")
+      @io.write([0,0].pack("NN"))
+    end
+
+    def encode_field(io, field, depth=0)
+      case field
+      when Integer
+        buf = [field].pack("N")
+        io.write([buf.bytesize].pack("N"))
+        io.write(buf)
+      when nil
+        io.write([-1].pack("N"))
+      when String
+        buf = field.encode("UTF-8")
+        io.write([buf.bytesize].pack("N"))
+        io.write(buf)
+      when Hash
+        raise Exception.new("Hash's can't contain hashes") if depth > 0
+        hash_io = StringIO.new
+        
+        hash_io.write([field.size].pack("N"))
+        field.each_pair {|key,val|
+          buf = key.to_s.encode("UTF-8")
+          hash_io.write([buf.bytesize].pack("N"))
+          hash_io.write(buf.to_s)
+          encode_field(hash_io, val.nil? ? val : val.to_s, depth + 1)
+        }
+        io.write([hash_io.pos].pack("N"))  # assumed identifier for hstore column
+        io.write(hash_io.string)
+      else
+        raise Exception.new("Unsupported Format: #{field.class.name}")
+      end
+    end
+
+  end
+end

data/lib/pg_data_encoder/version.rb

@@ -0,0 +1,3 @@
+module PgDataEncoder
+  VERSION = "0.0.1"
+end

data/lib/pg_data_encoder.rb

@@ -0,0 +1,7 @@
+require "pg_data_encoder/version"
+
+require 'pg_data_encoder/encode_for_copy'
+
+module PgDataEncoder
+  # Your code goes here...
+end

data/pg_data_encoder.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pg_data_encoder/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "pg_data_encoder"
+  gem.version       = PgDataEncoder::VERSION
+  gem.authors       = ["Pete Brumm"]
+  gem.email         = ["pete@petebrumm.com"]
+  gem.description   = %q{Creates a binary data file that can be imported into postgres's copy from command}
+  gem.summary       = %q{for faster input of data into postgres you can use this to generate the binary import and run COPY FROM}
+  gem.homepage      = "https://github.com/pbrumm/pg_data_encoder"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_development_dependency("rspec", ">= 2.12.0")
+  gem.add_development_dependency("rspec-core", ">= 2.12.0")
+end

data/spec/fixtures/3_col_hstore.txt

@@ -0,0 +1 @@
+1	text	"a"=>"1", "b"=>"asdf"

data/spec/spec_helper.rb

@@ -0,0 +1,21 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'rspec'
+require 'rspec/autorun'
+
+require 'pg_data_encoder'
+
+RSpec.configure do |config|
+  config.before(:suite) do
+
+
+  end
+end
+
+def filedata(filename)
+  str = nil
+  File.open("spec/fixtures/#{filename}", "r:ASCII-8BIT") {|io|
+    str = io.read
+  }
+  str
+end

data/spec/verify_data_formats_spec.rb

@@ -0,0 +1,30 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "generating data" do
+  it 'should encode hstore data correctly' do
+    encoder = PgDataEncoder::EncodeForCopy.new
+    encoder.add [1, "text", {a: 1, b: "asdf"}]
+    encoder.close
+    io = encoder.get_io
+    existing_data = filedata("3_col_hstore.dat")
+    str = io.read
+    io.class.name.should == "StringIO"
+    str.force_encoding("ASCII-8BIT")
+    #File.open("spec/fixtures/output.dat", "w:ASCII-8BIT") {|out| out.write(str) }
+    str.should == existing_data
+  end
+
+  it 'should encode hstore data correctly from tempfile' do
+    encoder = PgDataEncoder::EncodeForCopy.new(:use_tempfile => true)
+    encoder.add [1, "text", {a: 1, b: "asdf"}]
+    encoder.close
+    io = encoder.get_io
+    existing_data = filedata("3_col_hstore.dat")
+    str = io.read
+    io.class.name.should == "Tempfile"
+    str.force_encoding("ASCII-8BIT")
+    #File.open("spec/fixtures/output.dat", "w:ASCII-8BIT") {|out| out.write(str) }
+    str.should == existing_data
+  end
+
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: pg_data_encoder
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Pete Brumm
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.12.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.12.0
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.12.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.12.0
+description: Creates a binary data file that can be imported into postgres's copy
+  from command
+email:
+- pete@petebrumm.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/pg_data_encoder.rb
+- lib/pg_data_encoder/encode_for_copy.rb
+- lib/pg_data_encoder/version.rb
+- pg_data_encoder.gemspec
+- spec/fixtures/3_col_hstore.dat
+- spec/fixtures/3_col_hstore.txt
+- spec/fixtures/output.dat
+- spec/spec_helper.rb
+- spec/verify_data_formats_spec.rb
+homepage: https://github.com/pbrumm/pg_data_encoder
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: for faster input of data into postgres you can use this to generate the binary
+  import and run COPY FROM
+test_files:
+- spec/fixtures/3_col_hstore.dat
+- spec/fixtures/3_col_hstore.txt
+- spec/fixtures/output.dat
+- spec/spec_helper.rb
+- spec/verify_data_formats_spec.rb