DanaDanger-data_transport 0.2 → 0.3

data/lib/data_transport/data_store/active_record.rb

@@ -4,9 +4,6 @@ module DataTransport
     # This class is specifically optimized for reading and writing large
     # numbers of records, providing a significant advantage over using
     # ActiveRecord directly.
-    # 
-    # On MySQL databases, records are written in batches of the largest size
-    # possible instead of being inserted one by one.
     class ActiveRecord < DataStore
       # There are two ways to initialize this data store. The first is by
       # specifying one of your ActiveRecord models:
@@ -27,28 +24,40 @@ module DataTransport
       #
       # In addition, the following options are accepted:
       # 
-      # conditions::    Conditions describing which records to read. This can
-      #                 be anything that ActiveRecord will recognize, such as
-      #                 a hash table, an array with substitutions, or raw SQL.
-      #                 Default is nil (no conditions, read all records).
-      # truncate::      If true, the table will be truncated before any records
-      #                 are written. On MySQL databases, this is performed by
-      #                 executing a TRUNCATE TABLE query; all other databases
-      #                 use ActiveRecord's delete_all method.
-      # ignore_errors:: If true, errors that occur during record insertion will
-      #                 be ignored. This is useful if your table has a unique
-      #                 index and you want to silently drop records with
-      #                 duplicate keys. Currently this only works on MySQL.
-      #                 Default is false.
+      # conditions::     Conditions describing which records to read. This can
+      #                  be anything that ActiveRecord will recognize, such as
+      #                  a hash table, an array with substitutions, or raw SQL.
+      #                  Default is nil (no conditions, read all records).
+      # truncate::       If true, the table will be truncated before any
+      #                  records are written. On databases that support it,
+      #                  this is performed by executing a TRUNCATE TABLE query;
+      #                  all other databases use ActiveRecord's delete_all
+      #                  method.
+      # ignore_errors::  If true, errors that occur during record insertion
+      #                  will be ignored. This is useful if your table has a
+      #                  unique index and you want to silently drop records
+      #                  with duplicate keys. Currently this only works on
+      #                  MySQL. Default is false.
+      # max_sql_length:: Maximum permissible length of an SQL query, in bytes.
+      #                  Rows to be inserted are buffered until the largest
+      #                  possible INSERT statement has been generated, at which
+      #                  point the statement is executed and a new INSERT
+      #                  statement begins. The default value varies depending
+      #                  on what type of database you're connected to. With
+      #                  SQLite, the default is 1,000,000. With MySQL, the
+      #                  default is the value of the +max_allowed_packet+
+      #                  variable minus 512. With all other databases, the
+      #                  default is 16,777,216.
       def initialize(options = {})
         super()
         # Extract options.
-        @class         = options.delete(:class)
-        @connection    = options.delete(:connection)
-        @table_name    = options.delete(:table_name)
-        @conditions    = options.delete(:conditions)
-        @truncate      = options.delete(:truncate)
-        @ignore_errors = options.delete(:ignore_errors)
+        @class          = options.delete(:class)
+        @connection     = options.delete(:connection)
+        @table_name     = options.delete(:table_name)
+        @conditions     = options.delete(:conditions)
+        @truncate       = options.delete(:truncate)
+        @ignore_errors  = options.delete(:ignore_errors)
+        @max_sql_length = options.delete(:max_sql_length)
         # Make sure a class or connection and table name was provided.
         if @class.nil? && (@connection.nil? || @table_name.nil?)
           raise(ArgumentError, "missing required option `class', or `connection' and `table_name'")
@@ -83,12 +92,19 @@ module DataTransport
         unless options.empty?
           raise(ArgumentError, "unrecognized options: `#{options.join("', `")}'")
         end
-        # Ask the database how much data it can handle in one query. This only
-        # works on MySQL.
-        begin
-          rows = @class.connection.select_all("SHOW VARIABLES LIKE 'max_allowed_packet'")
-          @max_allowed_packet = rows.first["Value"].to_i - 512
-        rescue
+        # Figure out how much data the database can handle in one query.
+        if @max_sql_length
+          @max_sql_length = @max_sql_length.to_i
+        else
+          case @class.connection
+          when ::ActiveRecord::ConnectionAdapters::MysqlAdapter
+            rows = @class.connection.select_all("SHOW VARIABLES LIKE 'max_allowed_packet'")
+            @max_sql_length = rows.first["Value"].to_i - 512
+          when ::ActiveRecord::ConnectionAdapters::SQLiteAdapter
+            @max_sql_length = 1_000_000
+          else
+            @max_sql_length = 16_777_216
+          end
         end
         # Fetch column information
         @columns = {}
@@ -139,12 +155,12 @@ module DataTransport
         record.each {|k, v| values << conn.quote(v, @columns[k])}
         values = "(#{values.join ","}),"
         # Write the record.
-        if @max_allowed_packet.nil?
+        if @max_sql_length.nil?
           # We have no information on the database's maximum allowed packet
           # size, so it's safest to write the record immediately.
           @sql_buffer << values
           finalize
-        elsif @sql_buffer.length + record.length > @max_allowed_packet
+        elsif @sql_buffer.length + record.length > @max_sql_length
           # Appending this record to the SQL buffer will exceed the maximum
           # allowed packet size. Send the buffer to the database and start a
           # new statement with this record.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: DanaDanger-data_transport
 version: !ruby/object:Gem::Version 
-  version: "0.2"
+  version: "0.3"
 platform: ruby
 authors: 
 - Dana Danger