activerecord-jdbc-import 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +39 -0
- data/lib/active_record/jdbc/import.rb +101 -27
- data/lib/active_record/jdbc/import/version.rb +1 -1
- data/spec/import_spec.rb +36 -0
- metadata +2 -2
data/README.md
CHANGED
@@ -9,6 +9,9 @@ and we duck type, so we just want to say 'bulk load this chunk of data'.
|
|
9
9
|
|
10
10
|
This library aims to make loading data fast and easy. Just call a method, and it loads your data.
|
11
11
|
|
12
|
+
Note: The code here works, but it is pretty ugly. I am still working
|
13
|
+
through the implementation. This is a playground right now.
|
14
|
+
|
12
15
|
## License
|
13
16
|
|
14
17
|
MIT. Do what you want. If you make changes please contribute them back.
|
@@ -46,12 +49,48 @@ Or install it yourself as:
|
|
46
49
|
Product.import(products)
|
47
50
|
|
48
51
|
Product.count.should eq(100)
|
52
|
+
|
53
|
+
Or, you might want to use plain hashes. Creating an ActiveRecord object
|
54
|
+
for each row might be too much overhead for you:
|
55
|
+
|
56
|
+
require 'active_record/jdbc/import'
|
57
|
+
|
58
|
+
class Product < ActiveRecord::Base
|
59
|
+
include ActiveRecord::Jdbc::Import
|
60
|
+
end
|
61
|
+
|
62
|
+
products = []
|
49
63
|
|
64
|
+
1.upto(100) do
|
65
|
+
product = {
|
66
|
+
:name => "foobar123"
|
67
|
+
}
|
68
|
+
products << product
|
69
|
+
end
|
70
|
+
|
71
|
+
Product.import(products)
|
72
|
+
|
73
|
+
Product.count.should eq(100)
|
74
|
+
|
50
75
|
It is easy to use, and probably could be easier still. Feel free to fork the code,
|
51
76
|
make changes, fix bugs, etc.
|
52
77
|
|
78
|
+
## Tricks
|
79
|
+
|
80
|
+
If you are using MySQL, you can speed up the import by adding the
|
81
|
+
following options to your database.yml file:
|
82
|
+
|
83
|
+
development:
|
84
|
+
adapter: mysql
|
85
|
+
...
|
86
|
+
options:
|
87
|
+
useServerPrepStmts: 'false'
|
88
|
+
rewriteBatchedStatements: 'true'
|
89
|
+
|
53
90
|
## TODO
|
54
91
|
|
92
|
+
* Support more column types. Right now, this gem expects text and
|
93
|
+
numeric fields.
|
55
94
|
* The `id` column is currently ignored. The library assumes that `id` is going to be autoincremented. Make this optional.
|
56
95
|
* Test with more databases. Right now, Teradata, MySQL, and SQLite3 are all working.
|
57
96
|
|
@@ -10,41 +10,103 @@ module ActiveRecord
|
|
10
10
|
|
11
11
|
end
|
12
12
|
|
13
|
+
# Code here is ugly.
|
14
|
+
# Going to clean up as soon as the whole idea is worked out.
|
15
|
+
|
13
16
|
module ClassMethods
|
17
|
+
|
18
|
+
def import_infile(models)
|
19
|
+
return unless models.size > 0
|
20
|
+
|
21
|
+
start = Time.now
|
22
|
+
|
23
|
+
CSV.open("/tmp/upload.csv", "w") do |row|
|
24
|
+
|
25
|
+
row << models.first.keys.map { |a| a.to_s }
|
26
|
+
|
27
|
+
models.each do |model|
|
28
|
+
row << model.values
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
sql = []
|
33
|
+
sql << "LOAD DATA INFILE '/tmp/upload.csv'"
|
34
|
+
sql << "INTO TABLE #{self.table_name}"
|
35
|
+
sql << "FIELDS TERMINATED BY ','"
|
36
|
+
sql << "OPTIONALLY ENCLOSED BY '\"'"
|
37
|
+
sql << "LINES TERMINATED BY '\\n'"
|
38
|
+
sql << "IGNORE 1 LINES"
|
39
|
+
sql << "(#{models.first.keys.map { |a| a.to_s }.join(",")});"
|
40
|
+
|
41
|
+
self.connection.execute(sql.join(" "))
|
42
|
+
|
43
|
+
p "Wrote tempfile: #{Time.now - start}"
|
44
|
+
|
45
|
+
end
|
46
|
+
|
14
47
|
def import(models)
|
15
48
|
return unless models.size > 0
|
16
49
|
|
17
50
|
conn = self.connection.jdbc_connection
|
18
51
|
|
19
|
-
|
52
|
+
o = self.new
|
53
|
+
insert_sql = o.to_prepared_sql
|
54
|
+
ordered_columns = o.ordered_columns
|
20
55
|
|
21
56
|
pstmt = conn.prepareStatement(insert_sql)
|
22
57
|
|
23
58
|
conn.setAutoCommit(false)
|
24
59
|
|
25
|
-
models.
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
60
|
+
if models.first.is_a? Hash
|
61
|
+
models.each do |model|
|
62
|
+
i = 1
|
63
|
+
ordered_columns.each do |key|
|
64
|
+
next if key.to_s == 'id'
|
65
|
+
next if key.to_s == 'created_at'
|
66
|
+
next if key.to_s == 'updated_at'
|
67
|
+
|
68
|
+
column_type = self.columns_hash[key.to_s].type
|
69
|
+
value = model[key.to_sym]
|
70
|
+
|
71
|
+
if column_type == :integer and value.nil?
|
72
|
+
pstmt.setInt(i, nil)
|
73
|
+
elsif column_type == :integer
|
74
|
+
pstmt.setInt(i, value.to_i)
|
75
|
+
elsif key.to_s.downcase == 'week_date'
|
76
|
+
pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
|
77
|
+
elsif column_type == :string and value.nil?
|
78
|
+
pstmt.setString(i, nil)
|
79
|
+
else
|
80
|
+
pstmt.setString(i, value.to_s)
|
81
|
+
end
|
82
|
+
i += 1
|
42
83
|
end
|
43
|
-
|
44
|
-
|
84
|
+
pstmt.addBatch()
|
85
|
+
end
|
86
|
+
else
|
87
|
+
models.each do |model|
|
88
|
+
i = 1
|
89
|
+
model.attributes.each_pair do |key, value|
|
90
|
+
next if key.to_s == 'id'
|
91
|
+
next if key.to_s == 'created_at'
|
92
|
+
next if key.to_s == 'updated_at'
|
93
|
+
|
94
|
+
column_type = self.columns_hash[key.to_s].type
|
95
|
+
if column_type == :integer and value.nil?
|
96
|
+
pstmt.setInt(i, nil)
|
97
|
+
elsif column_type == :integer
|
98
|
+
pstmt.setInt(i, value.to_i)
|
99
|
+
elsif key.to_s.downcase == 'week_date'
|
100
|
+
pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
|
101
|
+
elsif column_type == :string and value.nil?
|
102
|
+
pstmt.setString(i, nil)
|
103
|
+
else
|
104
|
+
pstmt.setString(i, value.to_s)
|
105
|
+
end
|
106
|
+
i += 1
|
107
|
+
end
|
108
|
+
pstmt.addBatch()
|
45
109
|
end
|
46
|
-
|
47
|
-
pstmt.addBatch()
|
48
110
|
end
|
49
111
|
|
50
112
|
pstmt.executeBatch()
|
@@ -55,23 +117,35 @@ module ActiveRecord
|
|
55
117
|
end
|
56
118
|
end
|
57
119
|
|
120
|
+
def ordered_columns
|
121
|
+
@ordered_columns
|
122
|
+
end
|
123
|
+
|
58
124
|
def to_prepared_sql
|
59
125
|
conn = self.connection
|
60
|
-
|
126
|
+
|
127
|
+
at_date = DateTime.now
|
128
|
+
|
61
129
|
quoted_columns = []
|
62
130
|
quoted_values = []
|
131
|
+
@ordered_columns = []
|
63
132
|
attributes_with_values = self.send(:arel_attributes_values, true, true)
|
64
133
|
attributes_with_values.each_pair do |key,value|
|
65
134
|
next if key.name.to_s == 'id'
|
66
135
|
quoted_columns << conn.quote_column_name(key.name)
|
67
|
-
|
136
|
+
if key.name.to_s == 'created_at' or key.name.to_s == 'updated_at'
|
137
|
+
quoted_values << "'#{at_date.to_s(:db)}'"
|
138
|
+
else
|
139
|
+
@ordered_columns << key.name.to_s
|
140
|
+
quoted_values << '?'
|
141
|
+
end
|
68
142
|
end
|
69
|
-
|
143
|
+
|
70
144
|
"INSERT INTO #{self.class.quoted_table_name} " +
|
71
145
|
"(#{quoted_columns.join(', ')}) " +
|
72
|
-
"VALUES (#{quoted_values.join(', ')})
|
146
|
+
"VALUES (#{quoted_values.join(', ')})"
|
73
147
|
end
|
74
|
-
|
148
|
+
|
75
149
|
end
|
76
150
|
end
|
77
151
|
end
|
data/spec/import_spec.rb
CHANGED
@@ -27,6 +27,7 @@ describe ActiveRecord::Jdbc::Import do
|
|
27
27
|
end
|
28
28
|
|
29
29
|
it 'should import 100 rows of data' do
|
30
|
+
Product.delete_all
|
30
31
|
products = []
|
31
32
|
1.upto(100) do
|
32
33
|
product = Product.new
|
@@ -42,6 +43,41 @@ describe ActiveRecord::Jdbc::Import do
|
|
42
43
|
|
43
44
|
end
|
44
45
|
|
46
|
+
it 'should import hash data' do
|
47
|
+
Product.delete_all
|
48
|
+
products = []
|
49
|
+
1.upto(100) do
|
50
|
+
product = {
|
51
|
+
:code => Faker::Lorem.word,
|
52
|
+
:name => Faker::Name.name,
|
53
|
+
:vendor => Faker::Name.name,
|
54
|
+
:price => "#{rand(1000)}.#{rand(99)}".to_f
|
55
|
+
}
|
56
|
+
products << product
|
57
|
+
end
|
58
|
+
|
59
|
+
Product.import(products)
|
60
|
+
Product.count.should eq(100)
|
61
|
+
end
|
62
|
+
|
63
|
+
it 'should import hash data - out of order' do
|
64
|
+
Product.delete_all
|
65
|
+
products = []
|
66
|
+
1.upto(100) do
|
67
|
+
product = {
|
68
|
+
:vendor => Faker::Name.name,
|
69
|
+
:name => Faker::Name.name,
|
70
|
+
:price => "#{rand(1000)}.#{rand(99)}".to_f,
|
71
|
+
:code => Faker::Lorem.word
|
72
|
+
}
|
73
|
+
products << product
|
74
|
+
end
|
75
|
+
|
76
|
+
Product.import(products)
|
77
|
+
Product.count.should eq(100)
|
78
|
+
end
|
79
|
+
|
80
|
+
|
45
81
|
after(:all) do
|
46
82
|
CreateProducts.down
|
47
83
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: activerecord-jdbc-import
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-05-
|
12
|
+
date: 2013-05-28 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|