multisert 0.0.0.alpha → 0.0.1

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/README.md

@@ -18,7 +18,70 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+Let's start with a table:
+
+```sql
+CREATE TABLE IF NOT EXISTS some_database.some_table (
+  field_1 int default null,
+  field_2 int default null,
+  field_3 int default null,
+  field_4 int default null
+);
+```
+
+Now let's say we want to insert 1,000,000 records after running the
+current iterator through `some_magical_calculation` into our table from above.
+Let's assume that `some_magical_calculation` takes a single integer input and
+returns an array of 4 values.
+
+```ruby
+(0..1_000_000).each do |i|
+  res = some_magical_calculation(i)
+  dbclient.query %[
+    INSERT INTO some_database.some_table (field_1, field_2, field_3, field_4)
+    VALUES (#{res[0]}, #{res[1]}, #{res[2]}, #{res[3]})]
+end
+```
+
+This works, but we can improve it's speed by bundling up inserts using
+`Multisert`:
+
+```ruby
+buffer = Multisert.new connection: dbclient,
+                       database:   'some_database',
+                       table:      'some_table',
+                       fields:     ['field_1', 'field_2', 'field_3', 'field_4']
+
+(0..1_000_000).each do |i|
+  res = some_magical_calculation(i)
+  buffer << res
+end
+buffer.flush!
+```
+
+We start by creating a new Multisert instance, providing the database
+connection, database and table, and fields as attributes. Next, as we get the
+results from `some_magical_calculation`, we shovel each into the Multisert
+instance. As we iterate through, the Multisert instance will build up the
+records and then flush itself to the specified database table when it hits an
+internal count (default is 10_000, but can be set via the `max_buffer_count`
+attribute). One last thing to note is the `buffer.flush!` at the end of the
+script. This ensures that any pending entries are written to the database table
+that were not automatically taken care of by the auto-flush that will kick in
+during the iteration.
+
+## FAQ
+
+### Packet Too Large / Connection Lost Errors
+
+You may run into the "Packet Too Large" error when attempting to run a
+multisert. This can comeback as this error explicitly or as a "Connection
+Lost" error, depending on your mysql client.
+
+To learn more, [read the documentation](http://dev.mysql.com/doc/refman/5.5/en//packet-too-large.html).
+
+If you need to you can adjust the buffer size by setting `max_buffer_count`
+attribute. Generally, 10,000 to 100,000 is a pretty good starting range.
 
 ## Contributing
 

data/Rakefile

@@ -1,2 +1,28 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec) do |t|
+    t.rspec_opts = '-b'
+  end
+
+  task default: :spec
+rescue LoadError
+  $stderr.puts "rspec not available, spec task not provided"
+end
+
+begin
+  require 'cane/rake_task'
+
+  desc "Run cane to check quality metrics"
+  Cane::RakeTask.new(:quality) do |cane|
+    cane.abc_max    = 10
+    cane.style_glob = "lib/**/*.rb"
+    cane.no_doc     = true
+  end
+
+  task :default => :quality
+rescue LoadError
+  warn "cane not available, quality task not provided."
+end

data/lib/multisert.rb

@@ -1,5 +1,80 @@
-require "multisert/version"
+class Multisert
+  MAX_BUFFER_COUNT_DEFAULT = 10_000
 
-module Multisert
-  # Your code goes here...
+  attr_accessor :connection
+  attr_accessor :database
+  attr_accessor :table
+  attr_accessor :fields
+  attr_writer   :max_buffer_count
+
+  def initialize attrs = {}
+    attrs.each do |attr, value|
+      self.send "#{attr}=", value
+    end
+  end
+
+  def fields
+    @fields ||= []
+  end
+
+  def entries
+    @entries ||= []
+  end
+
+  def << entry
+    entries << entry
+    flush! if flush_buffer?
+    entry
+  end
+
+  def flush!
+    return if buffer_empty?
+    @connection.query multisert_sql
+    reset_entries!
+  end
+
+  def max_buffer_count
+    @max_buffer_count || MAX_BUFFER_COUNT_DEFAULT
+  end
+
+private
+
+  def buffer_empty?
+    entries.empty?
+  end
+
+  def flush_buffer?
+    entries.count >= max_buffer_count
+  end
+
+  def reset_entries!
+    @entries = []
+  end
+
+  def multisert_sql
+    "#{multisert_preamble} #{multisert_values}"
+  end
+
+  def multisert_preamble
+    "INSERT INTO #{database}.#{table} (#{fields.join(',')}) VALUES"
+  end
+
+  def multisert_values
+    @entries.reduce([]) { |memo, entries|
+      memo << "(#{entries.map { |e| cast e }.join(',')})"
+      memo
+    }.join(",")
+  end
+
+  def cast value
+    case value
+    when String
+      # TODO: want to escape the string too, checking for " and ;
+      "'#{value}'"
+    when Date
+      "'#{value}'"
+    else
+      value
+    end
+  end
 end

data/lib/multisert/version.rb

@@ -1,3 +1,3 @@
-module Multisert
-  VERSION = "0.0.0.alpha"
+class Multisert
+  VERSION = "0.0.1"
 end

data/multisert.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |gem|
   gem.email         = ["jeff.iacono@gmail.com"]
   gem.description   = %q{Buffer to handle bulk INSERTs}
   gem.summary       = %q{Buffer to handle bulk INSERTs}
-  gem.homepage      = ""
+  gem.homepage      = "https://github.com/jeffreyiacono/multisert"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -14,4 +14,9 @@ Gem::Specification.new do |gem|
   gem.name          = "multisert"
   gem.require_paths = ["lib"]
   gem.version       = Multisert::VERSION
+
+  gem.add_development_dependency "mysql2"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "cane"
+  gem.add_development_dependency "rspec", [">= 2"]
 end

data/spec/multisert_spec.rb

@@ -0,0 +1,151 @@
+require 'mysql2'
+require './spec/spec_helper'
+require './lib/multisert'
+
+# TODO: allow overriding in yaml config
+TEST_DATABASE = 'multisert_test'
+TEST_TABLE    = 'test_data'
+
+# TODO: make into yaml config
+$connection = Mysql2::Client.new(host: 'localhost', username: 'root')
+
+$cleaner = MrClean.new(database: TEST_DATABASE, connection: $connection) do |mgr|
+  mgr.create_table_schemas << %[
+    CREATE TABLE IF NOT EXISTS #{mgr.database}.#{TEST_TABLE} (
+      test_field_int_1 int default null,
+      test_field_int_2 int default null,
+      test_field_int_3 int default null,
+      test_field_int_4 int default null,
+      test_field_varchar varchar(10) default null,
+      test_field_date DATE default null
+    )]
+end
+
+describe Multisert do
+  describe "<<" do
+    let(:buffer) { described_class.new }
+
+    it "addes to the entries" do
+      buffer << [1, 2, 3]
+      buffer.entries.should == [[1, 2, 3]]
+    end
+
+    it "calls #flush! when the number of entries equals (or exceeds) max buffer count" do
+      buffer.max_buffer_count = 2
+      buffer.should_receive(:flush!)
+      buffer << [1, 2, 3]
+      buffer << [1, 2, 3]
+    end
+  end
+
+  describe "#flush!" do
+    let(:connection) { $connection }
+    let(:buffer) { described_class.new }
+
+    before do
+      $cleaner.ensure_clean_database! teardown_tables: (!!ENV['TEARDOWN'] || false)
+    end
+
+    it "does not fall over when there are no entries" do
+      flush_records = connection.query "DELETE FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      flush_records.to_a.should == []
+
+      buffer.flush!
+
+      flush_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      flush_records.to_a.should == []
+      buffer.entries.should == []
+    end
+
+    it "multi-inserts all added entries" do
+      pre_flush_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      pre_flush_records.to_a.should == []
+
+      buffer.connection = connection
+      buffer.database   = TEST_DATABASE
+      buffer.table      = TEST_TABLE
+      buffer.fields     = ['test_field_int_1',
+                           'test_field_int_2',
+                           'test_field_int_3',
+                           'test_field_int_4']
+
+      buffer << [ 1,  3,  4,  5]
+      buffer << [ 6,  7,  8,  9]
+      buffer << [10, 11, 12, 13]
+      buffer << [14, 15, 16, 17]
+
+      buffer.flush!
+
+      post_flush_records = connection.query %[
+        SELECT
+            test_field_int_1
+          , test_field_int_2
+          , test_field_int_3
+          , test_field_int_4
+        FROM #{TEST_DATABASE}.#{TEST_TABLE}]
+
+      post_flush_records.to_a.should == [
+        {'test_field_int_1' => 1,  'test_field_int_2' => 3,  'test_field_int_3' => 4,  'test_field_int_4' => 5},
+        {'test_field_int_1' => 6,  'test_field_int_2' => 7,  'test_field_int_3' => 8,  'test_field_int_4' => 9},
+        {'test_field_int_1' => 10, 'test_field_int_2' => 11, 'test_field_int_3' => 12, 'test_field_int_4' => 13},
+        {'test_field_int_1' => 14, 'test_field_int_2' => 15, 'test_field_int_3' => 16, 'test_field_int_4' => 17}]
+
+      buffer.entries.should == []
+    end
+
+    it "works with strings" do
+      pre_flush_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      pre_flush_records.to_a.should == []
+
+      buffer.connection = connection
+      buffer.database   = TEST_DATABASE
+      buffer.table      = TEST_TABLE
+      buffer.fields     = ['test_field_varchar']
+
+      buffer << ['a']
+      buffer << ['b']
+      buffer << ['c']
+      buffer << ['d']
+
+      buffer.flush!
+
+      post_flush_records = connection.query %[SELECT test_field_varchar FROM #{TEST_DATABASE}.#{TEST_TABLE}]
+      post_flush_records.to_a.should == [
+        {'test_field_varchar' => 'a'},
+        {'test_field_varchar' => 'b'},
+        {'test_field_varchar' => 'c'},
+        {'test_field_varchar' => 'd'}]
+
+      buffer.entries.should == []
+    end
+
+    it "works with strings that have illegal characters"
+
+    it "works with dates" do
+      pre_flush_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      pre_flush_records.to_a.should == []
+
+      buffer.connection = connection
+      buffer.database   = TEST_DATABASE
+      buffer.table      = TEST_TABLE
+      buffer.fields     = ['test_field_date']
+
+      buffer << [Date.new(2013, 1, 15)]
+      buffer << [Date.new(2013, 1, 16)]
+      buffer << [Date.new(2013, 1, 17)]
+      buffer << [Date.new(2013, 1, 18)]
+
+      buffer.flush!
+
+      post_flush_records = connection.query %[SELECT test_field_date FROM #{TEST_DATABASE}.#{TEST_TABLE}]
+
+      post_flush_records.to_a.should == [
+        {'test_field_date' => Date.parse('2013-01-15')},
+        {'test_field_date' => Date.parse('2013-01-16')},
+        {'test_field_date' => Date.parse('2013-01-17')},
+        {'test_field_date' => Date.parse('2013-01-18')}]
+
+      buffer.entries.should == []
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,46 @@
+class MrClean
+  attr_accessor :connection, :database, :create_table_schemas
+
+  def initialize attrs = {}
+    @connection           = attrs[:connection]
+    @database             = attrs[:database]
+    @create_table_schemas = attrs[:create_table_schemas] || []
+    yield self if block_given?
+  end
+
+  def ensure_clean_database! opts = {}
+    clean_database! !!opts[:teardown_tables]
+    ensure_tables!
+  end
+
+private
+
+  def database_exists?
+    @connection.query('show databases').to_a.map { |database|
+      database['Database']
+    }.include?(@database)
+  end
+
+  def ensure_database!
+    @connection.query "create database if not exists #{@database}"
+  end
+
+  def clean_database! teardown_tables
+    return unless database_exists?
+    @connection.query("show tables in #{@database}").to_a.each do |table|
+      if teardown_tables
+puts "TEARING DOWN"
+        @connection.query("drop table if exists #{@database}.#{table["Tables_in_#{@database}"]}")
+      else
+        @connection.query("truncate #{@database}.#{table["Tables_in_#{@database}"]}")
+      end
+    end
+  end
+
+  def ensure_tables!
+    ensure_database!
+    @create_table_schemas.each do |create_table_schema|
+      @connection.query create_table_schema
+    end
+  end
+end

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: multisert
 version: !ruby/object:Gem::Version
-  version: 0.0.0.alpha
-  prerelease: 6
+  version: 0.0.1
+  prerelease: 
 platform: ruby
 authors:
 - Jeff Iacono
@@ -10,7 +10,71 @@ autorequire:
 bindir: bin
 cert_chain: []
 date: 2013-03-07 00:00:00.000000000 Z
-dependencies: []
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cane
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2'
 description: Buffer to handle bulk INSERTs
 email:
 - jeff.iacono@gmail.com
@@ -19,6 +83,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rspec
 - Gemfile
 - LICENSE
 - README.md
@@ -26,7 +91,9 @@ files:
 - lib/multisert.rb
 - lib/multisert/version.rb
 - multisert.gemspec
-homepage: ''
+- spec/multisert_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/jeffreyiacono/multisert
 licenses: []
 post_install_message: 
 rdoc_options: []
@@ -41,13 +108,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>'
+  - - ! '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Buffer to handle bulk INSERTs
-test_files: []
+test_files:
+- spec/multisert_spec.rb
+- spec/spec_helper.rb