DanaDanger-data_transport 0.2 → 0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,9 +4,6 @@ module DataTransport
4
4
  # This class is specifically optimized for reading and writing large
5
5
  # numbers of records, providing a significant advantage over using
6
6
  # ActiveRecord directly.
7
- #
8
- # On MySQL databases, records are written in batches of the largest size
9
- # possible instead of being inserted one by one.
10
7
  class ActiveRecord < DataStore
11
8
  # There are two ways to initialize this data store. The first is by
12
9
  # specifying one of your ActiveRecord models:
@@ -27,28 +24,40 @@ module DataTransport
27
24
  #
28
25
  # In addition, the following options are accepted:
29
26
  #
30
- # conditions:: Conditions describing which records to read. This can
31
- # be anything that ActiveRecord will recognize, such as
32
- # a hash table, an array with substitutions, or raw SQL.
33
- # Default is nil (no conditions, read all records).
34
- # truncate:: If true, the table will be truncated before any records
35
- # are written. On MySQL databases, this is performed by
36
- # executing a TRUNCATE TABLE query; all other databases
37
- # use ActiveRecord's delete_all method.
38
- # ignore_errors:: If true, errors that occur during record insertion will
39
- # be ignored. This is useful if your table has a unique
40
- # index and you want to silently drop records with
41
- # duplicate keys. Currently this only works on MySQL.
42
- # Default is false.
27
+ # conditions:: Conditions describing which records to read. This can
28
+ # be anything that ActiveRecord will recognize, such as
29
+ # a hash table, an array with substitutions, or raw SQL.
30
+ # Default is nil (no conditions, read all records).
31
+ # truncate:: If true, the table will be truncated before any
32
+ # records are written. On databases that support it,
33
+ # this is performed by executing a TRUNCATE TABLE query;
34
+ # all other databases use ActiveRecord's delete_all
35
+ # method.
36
+ # ignore_errors:: If true, errors that occur during record insertion
37
+ # will be ignored. This is useful if your table has a
38
+ # unique index and you want to silently drop records
39
+ # with duplicate keys. Currently this only works on
40
+ # MySQL. Default is false.
41
+ # max_sql_length:: Maximum permissible length of an SQL query, in bytes.
42
+ # Rows to be inserted are buffered until the largest
43
+ # possible INSERT statement has been generated, at which
44
+ # point the statement is executed and a new INSERT
45
+ # statement begins. The default value varies depending
46
+ # on what type of database you're connected to. With
47
+ # SQLite, the default is 1,000,000. With MySQL, the
48
+ # default is the value of the +max_allowed_packet+
49
+ # variable minus 512. With all other databases, the
50
+ # default is 16,777,216.
43
51
  def initialize(options = {})
44
52
  super()
45
53
  # Extract options.
46
- @class = options.delete(:class)
47
- @connection = options.delete(:connection)
48
- @table_name = options.delete(:table_name)
49
- @conditions = options.delete(:conditions)
50
- @truncate = options.delete(:truncate)
51
- @ignore_errors = options.delete(:ignore_errors)
54
+ @class = options.delete(:class)
55
+ @connection = options.delete(:connection)
56
+ @table_name = options.delete(:table_name)
57
+ @conditions = options.delete(:conditions)
58
+ @truncate = options.delete(:truncate)
59
+ @ignore_errors = options.delete(:ignore_errors)
60
+ @max_sql_length = options.delete(:max_sql_length)
52
61
  # Make sure a class or connection and table name was provided.
53
62
  if @class.nil? && (@connection.nil? || @table_name.nil?)
54
63
  raise(ArgumentError, "missing required option `class', or `connection' and `table_name'")
@@ -83,12 +92,19 @@ module DataTransport
83
92
  unless options.empty?
84
93
  raise(ArgumentError, "unrecognized options: `#{options.join("', `")}'")
85
94
  end
86
- # Ask the database how much data it can handle in one query. This only
87
- # works on MySQL.
88
- begin
89
- rows = @class.connection.select_all("SHOW VARIABLES LIKE 'max_allowed_packet'")
90
- @max_allowed_packet = rows.first["Value"].to_i - 512
91
- rescue
95
+ # Figure out how much data the database can handle in one query.
96
+ if @max_sql_length
97
+ @max_sql_length = @max_sql_length.to_i
98
+ else
99
+ case @class.connection
100
+ when ::ActiveRecord::ConnectionAdapters::MysqlAdapter
101
+ rows = @class.connection.select_all("SHOW VARIABLES LIKE 'max_allowed_packet'")
102
+ @max_sql_length = rows.first["Value"].to_i - 512
103
+ when ::ActiveRecord::ConnectionAdapters::SQLiteAdapter
104
+ @max_sql_length = 1_000_000
105
+ else
106
+ @max_sql_length = 16_777_216
107
+ end
92
108
  end
93
109
  # Fetch column information
94
110
  @columns = {}
@@ -139,12 +155,12 @@ module DataTransport
139
155
  record.each {|k, v| values << conn.quote(v, @columns[k])}
140
156
  values = "(#{values.join ","}),"
141
157
  # Write the record.
142
- if @max_allowed_packet.nil?
158
+ if @max_sql_length.nil?
143
159
  # We have no information on the database's maximum allowed packet
144
160
  # size, so it's safest to write the record immediately.
145
161
  @sql_buffer << values
146
162
  finalize
147
- elsif @sql_buffer.length + record.length > @max_allowed_packet
163
+ elsif @sql_buffer.length + record.length > @max_sql_length
148
164
  # Appending this record to the SQL buffer will exceed the maximum
149
165
  # allowed packet size. Send the buffer to the database and start a
150
166
  # new statement with this record.
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: DanaDanger-data_transport
3
3
  version: !ruby/object:Gem::Version
4
- version: "0.2"
4
+ version: "0.3"
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dana Danger