multisert 0.0.2 → 0.0.3

data/README.md

@@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS some_database.some_table (
   field_2 int default null,
   field_3 int default null,
   field_4 int default null
-);
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
 ```
 
 Now let's say we want to insert 1,000,000 records after running the
@@ -57,22 +57,24 @@ buffer = Multisert.new connection: dbclient,
   res = some_magical_calculation(i)
   buffer << res
 end
-buffer.flush!
+buffer.write!
 ```
 
 We start by creating a new Multisert instance, providing the database
 connection, database and table, and fields as attributes. Next, as we get the
 results from `some_magical_calculation`, we shovel each into the Multisert
 instance. As we iterate through, the Multisert instance will build up the
-records and then flush itself to the specified database table when it hits an
+records and then write itself to the specified database table when it hits an
 internal count (default is 10_000, but can be set via the `max_buffer_count`
-attribute). One last thing to note is the `buffer.flush!` at the end of the
+attribute). One last thing to note is the `buffer.write!` at the end of the
 script. This ensures that any pending entries are written to the database table
-that were not automatically taken care of by the auto-flush that will kick in
+that were not automatically taken care of by the auto-write that will kick in
 during the iteration.
 
 ## Performance
 
+### Individual vs Buffer
+
 The gem has a quick performance test built in that can be run via:
 ```bash
 $ ruby ./performance/multisert_performance_test
@@ -117,6 +119,36 @@ The performance test was run on a computer with the following specs:
     L3 Cache:               3 MB
     Memory:                 4 GB
 
+All data was written to a mysql instance on localhost.
+
+### Buffer Sizes
+
+Let's take a look at how buffer size comes into play.
+
+We ran 3 separate and independent tests on the same computer as above.
+Additionally, also note that a buffer size of 0 and 1 are basically identical.
+
+If we look at using a buffer size ranging from 0 - 10, we see the following
+performance:
+
+<img src="https://raw.github.com/jeffreyiacono/images/master/multisert/multisert-performance-test-0-10.png" width="900" alt="Buffer size: 0 - 10" />
+
+If we take a step back and look at buffer sizes ranging from 0 - 100, we see the
+following performance:
+
+<img src="https://raw.github.com/jeffreyiacono/images/master/multisert/multisert-performance-test-0-100.png" width="900" alt="Buffer size: 0 - 100" />
+
+Finally, if we look at buffer sizes ranging from 0 - 1,000 and 0 - 10,000 we see
+the following performance (spoiler alert: not much difference, just more data
+points!):
+
+<img src="https://raw.github.com/jeffreyiacono/images/master/multisert/multisert-performance-test-0-1000.png" width="900" alt="Buffer size: 0 - 100" />
+
+<img src="https://raw.github.com/jeffreyiacono/images/master/multisert/multisert-performance-test-0-10000.png" width="900" alt="Buffer size: 0 - 100" />
+
+As can be seen, we see vastly improved performance as we increment our buffer
+from 0 - 100, but then level off thereafter.
+
 ## FAQ
 
 ### Packet Too Large / Connection Lost Errors

data/lib/multisert.rb

@@ -18,37 +18,44 @@ class Multisert
   end
 
   def entries
-    @entries ||= []
+    buffer
   end
 
   def << entry
     entries << entry
-    flush! if flush_buffer?
+    write_buffer! if write_buffer?
     entry
   end
 
-  def flush!
+  def write_buffer!
     return if buffer_empty?
     @connection.query multisert_sql
-    reset_entries!
+    reset_buffer!
   end
 
+  alias_method :write!, :write_buffer!
+  alias_method :flush!, :write_buffer!
+
   def max_buffer_count
     @max_buffer_count || MAX_BUFFER_COUNT_DEFAULT
   end
 
 private
 
-  def buffer_empty?
-    entries.empty?
+  def buffer
+    @buffer ||= []
   end
 
-  def flush_buffer?
-    entries.count >= max_buffer_count
+  def reset_buffer!
+    @buffer = []
+  end
+
+  def buffer_empty?
+    buffer.empty?
   end
 
-  def reset_entries!
-    @entries = []
+  def write_buffer?
+    buffer.count >= max_buffer_count
   end
 
   def multisert_sql
@@ -60,7 +67,7 @@ private
   end
 
   def multisert_values
-    @entries.reduce([]) { |memo, entries|
+    @buffer.reduce([]) { |memo, entries|
       memo << "(#{entries.map { |e| cast e }.join(',')})"
       memo
     }.join(",")

data/lib/multisert/version.rb

@@ -1,3 +1,3 @@
 class Multisert
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/multisert.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{Buffer to handle bulk INSERTs}
   gem.homepage      = "https://github.com/jeffreyiacono/multisert"
 
-  gem.files         = `git ls-files`.split($\)
+  gem.files         = `git ls-files`.split($\).delete_if { |f| f =~ /^data\// }
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "multisert"

data/performance/multisert_performance_test.rb

@@ -79,7 +79,14 @@ sample_records = generate_records
 
 puts_with_time "starting performance test: using #{sample_records.count} random entries, writing to #{PERFORMANCE_DESTINATION}"
 
-#insert_performance_test     CONNECTION, cleaner, sample_records, PERFORMANCE_DESTINATION
-(0..10_000).step(10) do |i|
+# individual insert vs multisert
+insert_performance_test     CONNECTION, cleaner, sample_records, PERFORMANCE_DESTINATION
+multinsert_performance_test CONNECTION, cleaner, sample_records, PERFORMANCE_DESTINATION, 10_000
+
+mini_steps = (0..9)
+big_steps  = (10..10_000).step(10)
+
+# buffer size performance test
+[*mini_steps, *big_steps].each do |i|
   multinsert_performance_test CONNECTION, cleaner, sample_records, PERFORMANCE_DESTINATION, i
 end

data/spec/multisert_spec.rb

@@ -28,18 +28,18 @@ describe Multisert do
 
     it "addes to the entries" do
       buffer << [1, 2, 3]
-      buffer.entries.should == [[1, 2, 3]]
+      expect(buffer.entries).to eq [[1, 2, 3]]
     end
 
     it "calls #flush! when the number of entries equals (or exceeds) max buffer count" do
       buffer.max_buffer_count = 2
-      buffer.should_receive(:flush!)
+      buffer.should_receive(:write_buffer!)
       buffer << [1, 2, 3]
       buffer << [1, 2, 3]
     end
   end
 
-  describe "#flush!" do
+  describe "#write_buffer!" do
     let(:connection) { $connection }
     let(:buffer) { described_class.new }
 
@@ -48,19 +48,19 @@ describe Multisert do
     end
 
     it "does not fall over when there are no entries" do
-      flush_records = connection.query "DELETE FROM #{TEST_DATABASE}.#{TEST_TABLE}"
-      flush_records.to_a.should == []
+      write_buffer_records = connection.query "DELETE FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      expect(write_buffer_records.to_a).to eq []
 
-      buffer.flush!
+      buffer.write_buffer!
 
-      flush_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
-      flush_records.to_a.should == []
-      buffer.entries.should == []
+      write_buffer_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      expect(write_buffer_records.to_a).to eq []
+      expect(buffer.entries).to eq []
     end
 
     it "multi-inserts all added entries" do
-      pre_flush_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
-      pre_flush_records.to_a.should == []
+      pre_write_buffer_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      expect(pre_write_buffer_records.to_a).to eq []
 
       buffer.connection = connection
       buffer.database   = TEST_DATABASE
@@ -75,9 +75,9 @@ describe Multisert do
       buffer << [10, 11, 12, 13]
       buffer << [14, 15, 16, 17]
 
-      buffer.flush!
+      buffer.write_buffer!
 
-      post_flush_records = connection.query %[
+      post_write_buffer_records = connection.query %[
         SELECT
             test_field_int_1
           , test_field_int_2
@@ -85,18 +85,19 @@ describe Multisert do
           , test_field_int_4
         FROM #{TEST_DATABASE}.#{TEST_TABLE}]
 
-      post_flush_records.to_a.should == [
+      expect(post_write_buffer_records.to_a).to eq [
         {'test_field_int_1' => 1,  'test_field_int_2' => 3,  'test_field_int_3' => 4,  'test_field_int_4' => 5},
         {'test_field_int_1' => 6,  'test_field_int_2' => 7,  'test_field_int_3' => 8,  'test_field_int_4' => 9},
         {'test_field_int_1' => 10, 'test_field_int_2' => 11, 'test_field_int_3' => 12, 'test_field_int_4' => 13},
-        {'test_field_int_1' => 14, 'test_field_int_2' => 15, 'test_field_int_3' => 16, 'test_field_int_4' => 17}]
+        {'test_field_int_1' => 14, 'test_field_int_2' => 15, 'test_field_int_3' => 16, 'test_field_int_4' => 17}
+      ]
 
-      buffer.entries.should == []
+      expect(buffer.entries).to eq []
     end
 
     it "works with strings" do
-      pre_flush_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
-      pre_flush_records.to_a.should == []
+      pre_write_buffer_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      expect(pre_write_buffer_records.to_a).to eq []
 
       buffer.connection = connection
       buffer.database   = TEST_DATABASE
@@ -108,23 +109,24 @@ describe Multisert do
       buffer << ['c']
       buffer << ['d']
 
-      buffer.flush!
+      buffer.write_buffer!
 
-      post_flush_records = connection.query %[SELECT test_field_varchar FROM #{TEST_DATABASE}.#{TEST_TABLE}]
-      post_flush_records.to_a.should == [
+      post_write_buffer_records = connection.query %[SELECT test_field_varchar FROM #{TEST_DATABASE}.#{TEST_TABLE}]
+      expect(post_write_buffer_records.to_a).to eq [
         {'test_field_varchar' => 'a'},
         {'test_field_varchar' => 'b'},
         {'test_field_varchar' => 'c'},
-        {'test_field_varchar' => 'd'}]
+        {'test_field_varchar' => 'd'}
+      ]
 
-      buffer.entries.should == []
+      expect(buffer.entries).to eq []
     end
 
     it "works with strings that have illegal characters"
 
     it "works with dates" do
-      pre_flush_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
-      pre_flush_records.to_a.should == []
+      pre_write_buffer_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      expect(pre_write_buffer_records.to_a).to eq []
 
       buffer.connection = connection
       buffer.database   = TEST_DATABASE
@@ -136,22 +138,23 @@ describe Multisert do
       buffer << [Date.new(2013, 1, 17)]
       buffer << [Date.new(2013, 1, 18)]
 
-      buffer.flush!
+      buffer.write_buffer!
 
-      post_flush_records = connection.query %[SELECT test_field_date FROM #{TEST_DATABASE}.#{TEST_TABLE}]
+      post_write_buffer_records = connection.query %[SELECT test_field_date FROM #{TEST_DATABASE}.#{TEST_TABLE}]
 
-      post_flush_records.to_a.should == [
+      expect(post_write_buffer_records.to_a).to eq [
         {'test_field_date' => Date.parse('2013-01-15')},
         {'test_field_date' => Date.parse('2013-01-16')},
         {'test_field_date' => Date.parse('2013-01-17')},
-        {'test_field_date' => Date.parse('2013-01-18')}]
+        {'test_field_date' => Date.parse('2013-01-18')}
+      ]
 
-      buffer.entries.should == []
+      expect(buffer.entries).to eq []
     end
 
     it "works with times" do
-      pre_flush_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
-      pre_flush_records.to_a.should == []
+      pre_write_buffer_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      expect(pre_write_buffer_records.to_a).to eq []
 
       buffer.connection = connection
       buffer.database   = TEST_DATABASE
@@ -163,17 +166,33 @@ describe Multisert do
       buffer << [Time.new(2013, 1, 17, 3, 7, 33)]
       buffer << [Time.new(2013, 1, 18, 4, 8, 44)]
 
-      buffer.flush!
+      buffer.write_buffer!
 
-      post_flush_records = connection.query %[SELECT test_field_datetime FROM #{TEST_DATABASE}.#{TEST_TABLE}]
+      post_write_buffer_records = connection.query %[SELECT test_field_datetime FROM #{TEST_DATABASE}.#{TEST_TABLE}]
 
-      post_flush_records.to_a.should == [
+      expect(post_write_buffer_records.to_a).to eq [
         {'test_field_datetime' => Time.new(2013, 1, 15, 1, 5, 11)},
         {'test_field_datetime' => Time.new(2013, 1, 16, 2, 6, 22)},
         {'test_field_datetime' => Time.new(2013, 1, 17, 3, 7, 33)},
         {'test_field_datetime' => Time.new(2013, 1, 18, 4, 8, 44)}]
 
-      buffer.entries.should == []
+      expect(buffer.entries).to eq []
+    end
+  end
+
+  describe "#flush!" do
+    it "aliases #write_buffer!" do
+      instance = described_class.new
+      flush_method = instance.method(:flush!)
+      expect(flush_method).to eq instance.method(:write_buffer!)
+    end
+  end
+
+  describe "#write!" do
+    it "aliases #write_buffer!" do
+      instance = described_class.new
+      flush_method = instance.method(:write!)
+      expect(flush_method).to eq instance.method(:write_buffer!)
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: multisert
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-09 00:00:00.000000000 Z
+date: 2013-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2