multisert 0.0.4 → 0.0.5

data/README.md

@@ -53,6 +53,23 @@ buffer = Multisert.new connection: dbclient,
                        table:      'some_table',
                        fields:     ['field_1', 'field_2', 'field_3', 'field_4']
 
+buffer.with_buffering do |buffer|
+  (0..1_000_000).each do |i|
+    res = some_magical_calculation(i)
+    buffer << res
+  end
+end
+```
+
+We start by creating a new Multisert instance, providing the database
+connection, database and table, and fields as attributes. Next, we leverage
+`#with_buffering` to wrap our sample iteration. Within the block, we shovel the
+results from `some_magical_calculation` into the Multisert instance, which then
+handles all the heavy lifting in terms of writing to the database.
+
+As an aside, `#with_buffering` is handling the following under the hood:
+
+```ruby
 (0..1_000_000).each do |i|
   res = some_magical_calculation(i)
   buffer << res
@@ -60,16 +77,13 @@ end
 buffer.write!
 ```
 
-We start by creating a new Multisert instance, providing the database
-connection, database and table, and fields as attributes. Next, as we get the
-results from `some_magical_calculation`, we shovel each into the Multisert
-instance. As we iterate through, the Multisert instance will build up the
-records and then write itself to the specified database table when it hits an
-internal count (default is 10_000, but can be set via the `max_buffer_count`
-attribute). One last thing to note is the `buffer.write!` at the end of the
-script. This ensures that any pending entries are written to the database table
-that were not automatically taken care of by the auto-write that will kick in
-during the iteration.
+As we iterate through, the Multisert instance will build up the records and
+then write itself to the specified database table when it hits an internal
+count (default is 10_000 entries, but this can be adjusted via the
+`max_buffer_count` attribute). The `buffer.write!` at the end ensures that
+any pending entries are written to the database table that were not
+automatically taken care of by the auto-write that will kick in during the
+iteration.
 
 ## Insert Strategies
 

data/lib/multisert.rb

@@ -43,6 +43,11 @@ class Multisert
     @max_buffer_count || MAX_BUFFER_COUNT_DEFAULT
   end
 
+  def with_buffering &block
+    yield self
+    write_buffer!
+  end
+
 private
 
   def insert_strategy?

data/lib/multisert/version.rb

@@ -1,3 +1,3 @@
 class Multisert
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/performance/multisert_performance_test.rb

@@ -38,7 +38,7 @@ def insert_performance_test connection, cleaner, sample_records, destination
   sample_records.each do |record|
     connection.query %[
       INSERT INTO #{destination} (#{fields})
-      VALUES (#{record.map { |k,v| v }.join(', ')})]
+      VALUES (#{record.values.join(', ')})]
   end
   runtime = timer.stop!
   ensure_data_completeness! connection, destination, sample_records.count
@@ -57,10 +57,11 @@ def multinsert_performance_test connection, cleaner, sample_records, destination
   cleaner.ensure_clean_database!
 
   (timer = Timer.new).start!
-  sample_records.each do |record|
-    buffer << record.map { |k, v| v }
+  buffer.with_buffering do |buffer|
+    sample_records.each do |record|
+      buffer << record.values
+    end
   end
-  buffer.flush!
   runtime = timer.stop!
   ensure_data_completeness! connection, destination, sample_records.count
   puts "multisert w/ buffer of #{buffer.max_buffer_count} took #{runtime.round(2)}s to insert #{sample_records.count} entries"

data/spec/multisert_spec.rb

@@ -7,12 +7,22 @@ TEST_DATABASE      = 'multisert_test'
 TEST_TABLE         = 'test_data'
 TEST_INDEXED_TABLE = 'test_indexed_data'
 
+def test_table
+  "#{TEST_DATABASE}.#{TEST_TABLE}"
+end
+
+def test_indexed_table
+  "#{TEST_DATABASE}.#{TEST_INDEXED_TABLE}"
+end
+
 # TODO: make into yaml config
 $connection = Mysql2::Client.new(host: 'localhost', username: 'root')
 
-$cleaner = MultisertSpec::MrClean.new(database: TEST_DATABASE, connection: $connection) do |mgr|
+$cleaner = MultisertSpec::MrClean.new(database: TEST_DATABASE,
+                                      connection: $connection) do |mgr|
+
   mgr.create_table_schemas << %[
-    CREATE TABLE IF NOT EXISTS #{mgr.database}.#{TEST_TABLE} (
+    CREATE TABLE IF NOT EXISTS #{test_table} (
       test_field_int_1 int default null,
       test_field_int_2 int default null,
       test_field_int_3 int default null,
@@ -22,7 +32,7 @@ $cleaner = MultisertSpec::MrClean.new(database: TEST_DATABASE, connection: $conn
       test_field_datetime DATETIME default null)]
 
   mgr.create_table_schemas << %[
-    CREATE TABLE IF NOT EXISTS #{mgr.database}.#{TEST_INDEXED_TABLE} (
+    CREATE TABLE IF NOT EXISTS #{test_indexed_table} (
       test_id int not null,
       test_field varchar(15) default null,
       primary key (test_id))]
@@ -54,17 +64,17 @@ describe Multisert do
     end
 
     it "does not fall over when there are no entries" do
-      connection.query "DELETE FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      connection.query "DELETE FROM #{test_table}"
 
       buffer.write_buffer!
 
-      write_buffer_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      write_buffer_records = connection.query "SELECT * FROM #{test_table}"
       expect(write_buffer_records.to_a).to eq []
       expect(buffer.entries).to eq []
     end
 
     it "multi-inserts all added entries and clears #entries" do
-      pre_write_buffer_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      pre_write_buffer_records = connection.query "SELECT * FROM #{test_table}"
       expect(pre_write_buffer_records.to_a).to eq []
 
       buffer.connection = connection
@@ -88,7 +98,7 @@ describe Multisert do
           , test_field_int_2
           , test_field_int_3
           , test_field_int_4
-        FROM #{TEST_DATABASE}.#{TEST_TABLE}]
+        FROM #{test_table}]
 
       expect(post_write_buffer_records.to_a).to eq [
         {'test_field_int_1' => 1,  'test_field_int_2' => 3,  'test_field_int_3' => 4,  'test_field_int_4' => 5},
@@ -100,7 +110,7 @@ describe Multisert do
     end
 
     it "works with strings" do
-      pre_write_buffer_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      pre_write_buffer_records = connection.query "SELECT * FROM #{test_table}"
       expect(pre_write_buffer_records.to_a).to eq []
 
       buffer.connection = connection
@@ -115,7 +125,7 @@ describe Multisert do
 
       buffer.write_buffer!
 
-      post_write_buffer_records = connection.query %[SELECT test_field_varchar FROM #{TEST_DATABASE}.#{TEST_TABLE}]
+      post_write_buffer_records = connection.query %[SELECT test_field_varchar FROM #{test_table}]
       expect(post_write_buffer_records.to_a).to eq [
         {'test_field_varchar' => 'a'},
         {'test_field_varchar' => 'b'},
@@ -128,7 +138,7 @@ describe Multisert do
     it "works with strings that have illegal characters"
 
     it "works with dates" do
-      pre_write_buffer_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      pre_write_buffer_records = connection.query "SELECT * FROM #{test_table}"
       expect(pre_write_buffer_records.to_a).to eq []
 
       buffer.connection = connection
@@ -143,7 +153,7 @@ describe Multisert do
 
       buffer.write_buffer!
 
-      post_write_buffer_records = connection.query %[SELECT test_field_date FROM #{TEST_DATABASE}.#{TEST_TABLE}]
+      post_write_buffer_records = connection.query %[SELECT test_field_date FROM #{test_table}]
 
       expect(post_write_buffer_records.to_a).to eq [
         {'test_field_date' => Date.parse('2013-01-15')},
@@ -155,7 +165,7 @@ describe Multisert do
     end
 
     it "works with times" do
-      pre_write_buffer_records = connection.query "SELECT * FROM #{TEST_DATABASE}.#{TEST_TABLE}"
+      pre_write_buffer_records = connection.query "SELECT * FROM #{test_table}"
       expect(pre_write_buffer_records.to_a).to eq []
 
       buffer.connection = connection
@@ -170,7 +180,7 @@ describe Multisert do
 
       buffer.write_buffer!
 
-      post_write_buffer_records = connection.query %[SELECT test_field_datetime FROM #{TEST_DATABASE}.#{TEST_TABLE}]
+      post_write_buffer_records = connection.query %[SELECT test_field_datetime FROM #{test_table}]
 
       expect(post_write_buffer_records.to_a).to eq [
         {'test_field_datetime' => Time.new(2013, 1, 15, 1, 5, 11)},
@@ -208,7 +218,7 @@ describe Multisert do
 
     context "set to replace" do
       it "writes over an existing record with the same primary / unique key" do
-        connection.query %[INSERT INTO #{TEST_DATABASE}.#{TEST_INDEXED_TABLE} (test_id, test_field)
+        connection.query %[INSERT INTO #{test_indexed_table} (test_id, test_field)
                            VALUES (1, 'ONE'), (2, 'TWO')]
 
         buffer.connection      = connection
@@ -218,6 +228,7 @@ describe Multisert do
         buffer.insert_strategy = :replace
 
         buffer << [1, 'SOMETHING NEW']
+        buffer << [3, 'ALSO NEW']
 
         buffer.write_buffer!
 
@@ -225,17 +236,18 @@ describe Multisert do
           SELECT
               test_id
             , test_field
-          FROM #{TEST_DATABASE}.#{TEST_INDEXED_TABLE}]
+          FROM #{test_indexed_table}]
 
         expect(post_write_buffer_records.to_a).to eq [
           {'test_id' => 1, 'test_field' => 'SOMETHING NEW'},
-          {'test_id' => 2, 'test_field' => 'TWO'}]
+          {'test_id' => 2, 'test_field' => 'TWO'},
+          {'test_id' => 3, 'test_field' => 'ALSO NEW'}]
       end
     end
 
     context "set to ignore" do
       before do
-        connection.query %[INSERT INTO #{TEST_DATABASE}.#{TEST_INDEXED_TABLE} (test_id, test_field)
+        connection.query %[INSERT INTO #{test_indexed_table} (test_id, test_field)
                            VALUES (1, 'ONE'), (2, 'TWO')]
 
         buffer.connection      = connection
@@ -245,6 +257,7 @@ describe Multisert do
         buffer.insert_strategy = :ignore
 
         buffer << [1, 'NEW']
+        buffer << [3, 'ALSO NEW']
       end
 
       it "does not raise an error" do
@@ -258,11 +271,12 @@ describe Multisert do
           SELECT
               test_id
             , test_field
-          FROM #{TEST_DATABASE}.#{TEST_INDEXED_TABLE}]
+          FROM #{test_indexed_table}]
 
         expect(post_write_buffer_records.to_a).to eq [
           {'test_id' => 1, 'test_field' => 'ONE'},
-          {'test_id' => 2, 'test_field' => 'TWO'}]
+          {'test_id' => 2, 'test_field' => 'TWO'},
+          {'test_id' => 3, 'test_field' => 'ALSO NEW'}]
       end
     end
 
@@ -280,4 +294,59 @@ describe Multisert do
       end
     end
   end
+
+  describe "#with_buffering" do
+    let(:connection) { $connection }
+    let(:buffer) { described_class.new }
+
+    before do
+      $cleaner.ensure_clean_database! teardown_tables: (!!ENV['TEARDOWN'] || false)
+    end
+
+    it "ensures all records are inserted" do
+      pre_write_buffer_records = connection.query "SELECT * FROM #{test_table}"
+      expect(pre_write_buffer_records.to_a).to eq []
+
+      buffer.connection = connection
+      buffer.database   = TEST_DATABASE
+      buffer.table      = TEST_TABLE
+      buffer.fields     = ['test_field_int_1',
+                           'test_field_int_2',
+                           'test_field_int_3',
+                           'test_field_int_4']
+
+
+
+      sample_entries = [[ 1,  3,  4,  5],
+                        [ 6,  7,  8,  9],
+                        [10, 11, 12, 13],
+                        [14, 15, 16, 17]]
+
+      # set this to 1 less than the total number of entries we want to buffer.
+      # test that all are written w/o explicitly #write!
+      buffer.max_buffer_count = sample_entries.length - 1
+
+      buffer.with_buffering do |buffer|
+        sample_entries.each do |entry|
+          buffer << entry.to_a
+        end
+      end
+
+      post_write_buffer_records = connection.query %[
+        SELECT
+            test_field_int_1
+          , test_field_int_2
+          , test_field_int_3
+          , test_field_int_4
+        FROM #{test_table}]
+
+      expect(post_write_buffer_records.to_a).to eq [
+        {'test_field_int_1' => 1,  'test_field_int_2' => 3,  'test_field_int_3' => 4,  'test_field_int_4' => 5},
+        {'test_field_int_1' => 6,  'test_field_int_2' => 7,  'test_field_int_3' => 8,  'test_field_int_4' => 9},
+        {'test_field_int_1' => 10, 'test_field_int_2' => 11, 'test_field_int_3' => 12, 'test_field_int_4' => 13},
+        {'test_field_int_1' => 14, 'test_field_int_2' => 15, 'test_field_int_3' => 16, 'test_field_int_4' => 17}]
+
+      expect(buffer.entries).to eq []
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: multisert
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-27 00:00:00.000000000 Z
+date: 2013-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2