DanaDanger-data_transport 0.2 → 0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/data_transport/data_store/active_record.rb +46 -30
- metadata +1 -1
@@ -4,9 +4,6 @@ module DataTransport
|
|
4
4
|
# This class is specifically optimized for reading and writing large
|
5
5
|
# numbers of records, providing a significant advantage over using
|
6
6
|
# ActiveRecord directly.
|
7
|
-
#
|
8
|
-
# On MySQL databases, records are written in batches of the largest size
|
9
|
-
# possible instead of being inserted one by one.
|
10
7
|
class ActiveRecord < DataStore
|
11
8
|
# There are two ways to initialize this data store. The first is by
|
12
9
|
# specifying one of your ActiveRecord models:
|
@@ -27,28 +24,40 @@ module DataTransport
|
|
27
24
|
#
|
28
25
|
# In addition, the following options are accepted:
|
29
26
|
#
|
30
|
-
# conditions::
|
31
|
-
#
|
32
|
-
#
|
33
|
-
#
|
34
|
-
# truncate::
|
35
|
-
#
|
36
|
-
#
|
37
|
-
#
|
38
|
-
#
|
39
|
-
#
|
40
|
-
#
|
41
|
-
#
|
42
|
-
#
|
27
|
+
# conditions:: Conditions describing which records to read. This can
|
28
|
+
# be anything that ActiveRecord will recognize, such as
|
29
|
+
# a hash table, an array with substitutions, or raw SQL.
|
30
|
+
# Default is nil (no conditions, read all records).
|
31
|
+
# truncate:: If true, the table will be truncated before any
|
32
|
+
# records are written. On databases that support it,
|
33
|
+
# this is performed by executing a TRUNCATE TABLE query;
|
34
|
+
# all other databases use ActiveRecord's delete_all
|
35
|
+
# method.
|
36
|
+
# ignore_errors:: If true, errors that occur during record insertion
|
37
|
+
# will be ignored. This is useful if your table has a
|
38
|
+
# unique index and you want to silently drop records
|
39
|
+
# with duplicate keys. Currently this only works on
|
40
|
+
# MySQL. Default is false.
|
41
|
+
# max_sql_length:: Maximum permissible length of an SQL query, in bytes.
|
42
|
+
# Rows to be inserted are buffered until the largest
|
43
|
+
# possible INSERT statement has been generated, at which
|
44
|
+
# point the statement is executed and a new INSERT
|
45
|
+
# statement begins. The default value varies depending
|
46
|
+
# on what type of database you're connected to. With
|
47
|
+
# SQLite, the default is 1,000,000. With MySQL, the
|
48
|
+
# default is the value of the +max_allowed_packet+
|
49
|
+
# variable minus 512. With all other databases, the
|
50
|
+
# default is 16,777,216.
|
43
51
|
def initialize(options = {})
|
44
52
|
super()
|
45
53
|
# Extract options.
|
46
|
-
@class
|
47
|
-
@connection
|
48
|
-
@table_name
|
49
|
-
@conditions
|
50
|
-
@truncate
|
51
|
-
@ignore_errors
|
54
|
+
@class = options.delete(:class)
|
55
|
+
@connection = options.delete(:connection)
|
56
|
+
@table_name = options.delete(:table_name)
|
57
|
+
@conditions = options.delete(:conditions)
|
58
|
+
@truncate = options.delete(:truncate)
|
59
|
+
@ignore_errors = options.delete(:ignore_errors)
|
60
|
+
@max_sql_length = options.delete(:max_sql_length)
|
52
61
|
# Make sure a class or connection and table name was provided.
|
53
62
|
if @class.nil? && (@connection.nil? || @table_name.nil?)
|
54
63
|
raise(ArgumentError, "missing required option `class', or `connection' and `table_name'")
|
@@ -83,12 +92,19 @@ module DataTransport
|
|
83
92
|
unless options.empty?
|
84
93
|
raise(ArgumentError, "unrecognized options: `#{options.join("', `")}'")
|
85
94
|
end
|
86
|
-
#
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
@
|
91
|
-
|
95
|
+
# Figure out how much data the database can handle in one query.
|
96
|
+
if @max_sql_length
|
97
|
+
@max_sql_length = @max_sql_length.to_i
|
98
|
+
else
|
99
|
+
case @class.connection
|
100
|
+
when ::ActiveRecord::ConnectionAdapters::MysqlAdapter
|
101
|
+
rows = @class.connection.select_all("SHOW VARIABLES LIKE 'max_allowed_packet'")
|
102
|
+
@max_sql_length = rows.first["Value"].to_i - 512
|
103
|
+
when ::ActiveRecord::ConnectionAdapters::SQLiteAdapter
|
104
|
+
@max_sql_length = 1_000_000
|
105
|
+
else
|
106
|
+
@max_sql_length = 16_777_216
|
107
|
+
end
|
92
108
|
end
|
93
109
|
# Fetch column information
|
94
110
|
@columns = {}
|
@@ -139,12 +155,12 @@ module DataTransport
|
|
139
155
|
record.each {|k, v| values << conn.quote(v, @columns[k])}
|
140
156
|
values = "(#{values.join ","}),"
|
141
157
|
# Write the record.
|
142
|
-
if @
|
158
|
+
if @max_sql_length.nil?
|
143
159
|
# We have no information on the database's maximum allowed packet
|
144
160
|
# size, so it's safest to write the record immediately.
|
145
161
|
@sql_buffer << values
|
146
162
|
finalize
|
147
|
-
elsif @sql_buffer.length + record.length > @
|
163
|
+
elsif @sql_buffer.length + record.length > @max_sql_length
|
148
164
|
# Appending this record to the SQL buffer will exceed the maximum
|
149
165
|
# allowed packet size. Send the buffer to the database and start a
|
150
166
|
# new statement with this record.
|