activerecord-jdbc-import 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +39 -0
- data/lib/active_record/jdbc/import.rb +101 -27
- data/lib/active_record/jdbc/import/version.rb +1 -1
- data/spec/import_spec.rb +36 -0
- metadata +2 -2
data/README.md
CHANGED
@@ -9,6 +9,9 @@ and we duck type, so we just want to say 'bulk load this chunk of data'.
|
|
9
9
|
|
10
10
|
This library aims to make loading data fast and easy. Just call a method, and it loads your data.
|
11
11
|
|
12
|
+
Note: The code here works, but it is pretty ugly. I am still working
|
13
|
+
through the implementation. This is a playground right now.
|
14
|
+
|
12
15
|
## License
|
13
16
|
|
14
17
|
MIT. Do what you want. If you make changes please contribute them back.
|
@@ -46,12 +49,48 @@ Or install it yourself as:
|
|
46
49
|
Product.import(products)
|
47
50
|
|
48
51
|
Product.count.should eq(100)
|
52
|
+
|
53
|
+
Or, you might want to use plain hashes. Creating an ActiveRecord object
|
54
|
+
for each row might be too much overhead for you:
|
55
|
+
|
56
|
+
require 'active_record/jdbc/import'
|
57
|
+
|
58
|
+
class Product < ActiveRecord::Base
|
59
|
+
include ActiveRecord::Jdbc::Import
|
60
|
+
end
|
61
|
+
|
62
|
+
products = []
|
49
63
|
|
64
|
+
1.upto(100) do
|
65
|
+
product = {
|
66
|
+
:name => "foobar123"
|
67
|
+
}
|
68
|
+
products << product
|
69
|
+
end
|
70
|
+
|
71
|
+
Product.import(products)
|
72
|
+
|
73
|
+
Product.count.should eq(100)
|
74
|
+
|
50
75
|
It is easy to use, and probably could be easier still. Feel free to fork the code,
|
51
76
|
make changes, fix bugs, etc.
|
52
77
|
|
78
|
+
## Tricks
|
79
|
+
|
80
|
+
If you are using MySQL, you can speed up the import by adding the
|
81
|
+
following options to your database.yml file:
|
82
|
+
|
83
|
+
development:
|
84
|
+
adapter: mysql
|
85
|
+
...
|
86
|
+
options:
|
87
|
+
useServerPrepStmts: 'false'
|
88
|
+
rewriteBatchedStatements: 'true'
|
89
|
+
|
53
90
|
## TODO
|
54
91
|
|
92
|
+
* Support more column types. Right now, this gem expects text and
|
93
|
+
numeric fields.
|
55
94
|
* The `id` column is currently ignored. The library assumes that `id` is going to be autoincremented. Make this optional.
|
56
95
|
* Test with more databases. Right now, Teradata, MySQL, and SQLite3 are all working.
|
57
96
|
|
@@ -10,41 +10,103 @@ module ActiveRecord
|
|
10
10
|
|
11
11
|
end
|
12
12
|
|
13
|
+
# Code here is ugly.
|
14
|
+
# Going to clean up as soon as the whole idea is worked out.
|
15
|
+
|
13
16
|
module ClassMethods
|
17
|
+
|
18
|
+
def import_infile(models)
|
19
|
+
return unless models.size > 0
|
20
|
+
|
21
|
+
start = Time.now
|
22
|
+
|
23
|
+
CSV.open("/tmp/upload.csv", "w") do |row|
|
24
|
+
|
25
|
+
row << models.first.keys.map { |a| a.to_s }
|
26
|
+
|
27
|
+
models.each do |model|
|
28
|
+
row << model.values
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
sql = []
|
33
|
+
sql << "LOAD DATA INFILE '/tmp/upload.csv'"
|
34
|
+
sql << "INTO TABLE #{self.table_name}"
|
35
|
+
sql << "FIELDS TERMINATED BY ','"
|
36
|
+
sql << "OPTIONALLY ENCLOSED BY '\"'"
|
37
|
+
sql << "LINES TERMINATED BY '\\n'"
|
38
|
+
sql << "IGNORE 1 LINES"
|
39
|
+
sql << "(#{models.first.keys.map { |a| a.to_s }.join(",")});"
|
40
|
+
|
41
|
+
self.connection.execute(sql.join(" "))
|
42
|
+
|
43
|
+
p "Wrote tempfile: #{Time.now - start}"
|
44
|
+
|
45
|
+
end
|
46
|
+
|
14
47
|
def import(models)
|
15
48
|
return unless models.size > 0
|
16
49
|
|
17
50
|
conn = self.connection.jdbc_connection
|
18
51
|
|
19
|
-
|
52
|
+
o = self.new
|
53
|
+
insert_sql = o.to_prepared_sql
|
54
|
+
ordered_columns = o.ordered_columns
|
20
55
|
|
21
56
|
pstmt = conn.prepareStatement(insert_sql)
|
22
57
|
|
23
58
|
conn.setAutoCommit(false)
|
24
59
|
|
25
|
-
models.
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
60
|
+
if models.first.is_a? Hash
|
61
|
+
models.each do |model|
|
62
|
+
i = 1
|
63
|
+
ordered_columns.each do |key|
|
64
|
+
next if key.to_s == 'id'
|
65
|
+
next if key.to_s == 'created_at'
|
66
|
+
next if key.to_s == 'updated_at'
|
67
|
+
|
68
|
+
column_type = self.columns_hash[key.to_s].type
|
69
|
+
value = model[key.to_sym]
|
70
|
+
|
71
|
+
if column_type == :integer and value.nil?
|
72
|
+
pstmt.setInt(i, nil)
|
73
|
+
elsif column_type == :integer
|
74
|
+
pstmt.setInt(i, value.to_i)
|
75
|
+
elsif key.to_s.downcase == 'week_date'
|
76
|
+
pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
|
77
|
+
elsif column_type == :string and value.nil?
|
78
|
+
pstmt.setString(i, nil)
|
79
|
+
else
|
80
|
+
pstmt.setString(i, value.to_s)
|
81
|
+
end
|
82
|
+
i += 1
|
42
83
|
end
|
43
|
-
|
44
|
-
|
84
|
+
pstmt.addBatch()
|
85
|
+
end
|
86
|
+
else
|
87
|
+
models.each do |model|
|
88
|
+
i = 1
|
89
|
+
model.attributes.each_pair do |key, value|
|
90
|
+
next if key.to_s == 'id'
|
91
|
+
next if key.to_s == 'created_at'
|
92
|
+
next if key.to_s == 'updated_at'
|
93
|
+
|
94
|
+
column_type = self.columns_hash[key.to_s].type
|
95
|
+
if column_type == :integer and value.nil?
|
96
|
+
pstmt.setInt(i, nil)
|
97
|
+
elsif column_type == :integer
|
98
|
+
pstmt.setInt(i, value.to_i)
|
99
|
+
elsif key.to_s.downcase == 'week_date'
|
100
|
+
pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
|
101
|
+
elsif column_type == :string and value.nil?
|
102
|
+
pstmt.setString(i, nil)
|
103
|
+
else
|
104
|
+
pstmt.setString(i, value.to_s)
|
105
|
+
end
|
106
|
+
i += 1
|
107
|
+
end
|
108
|
+
pstmt.addBatch()
|
45
109
|
end
|
46
|
-
|
47
|
-
pstmt.addBatch()
|
48
110
|
end
|
49
111
|
|
50
112
|
pstmt.executeBatch()
|
@@ -55,23 +117,35 @@ module ActiveRecord
|
|
55
117
|
end
|
56
118
|
end
|
57
119
|
|
120
|
+
def ordered_columns
|
121
|
+
@ordered_columns
|
122
|
+
end
|
123
|
+
|
58
124
|
def to_prepared_sql
|
59
125
|
conn = self.connection
|
60
|
-
|
126
|
+
|
127
|
+
at_date = DateTime.now
|
128
|
+
|
61
129
|
quoted_columns = []
|
62
130
|
quoted_values = []
|
131
|
+
@ordered_columns = []
|
63
132
|
attributes_with_values = self.send(:arel_attributes_values, true, true)
|
64
133
|
attributes_with_values.each_pair do |key,value|
|
65
134
|
next if key.name.to_s == 'id'
|
66
135
|
quoted_columns << conn.quote_column_name(key.name)
|
67
|
-
|
136
|
+
if key.name.to_s == 'created_at' or key.name.to_s == 'updated_at'
|
137
|
+
quoted_values << "'#{at_date.to_s(:db)}'"
|
138
|
+
else
|
139
|
+
@ordered_columns << key.name.to_s
|
140
|
+
quoted_values << '?'
|
141
|
+
end
|
68
142
|
end
|
69
|
-
|
143
|
+
|
70
144
|
"INSERT INTO #{self.class.quoted_table_name} " +
|
71
145
|
"(#{quoted_columns.join(', ')}) " +
|
72
|
-
"VALUES (#{quoted_values.join(', ')})
|
146
|
+
"VALUES (#{quoted_values.join(', ')})"
|
73
147
|
end
|
74
|
-
|
148
|
+
|
75
149
|
end
|
76
150
|
end
|
77
151
|
end
|
data/spec/import_spec.rb
CHANGED
@@ -27,6 +27,7 @@ describe ActiveRecord::Jdbc::Import do
|
|
27
27
|
end
|
28
28
|
|
29
29
|
it 'should import 100 rows of data' do
|
30
|
+
Product.delete_all
|
30
31
|
products = []
|
31
32
|
1.upto(100) do
|
32
33
|
product = Product.new
|
@@ -42,6 +43,41 @@ describe ActiveRecord::Jdbc::Import do
|
|
42
43
|
|
43
44
|
end
|
44
45
|
|
46
|
+
it 'should import hash data' do
|
47
|
+
Product.delete_all
|
48
|
+
products = []
|
49
|
+
1.upto(100) do
|
50
|
+
product = {
|
51
|
+
:code => Faker::Lorem.word,
|
52
|
+
:name => Faker::Name.name,
|
53
|
+
:vendor => Faker::Name.name,
|
54
|
+
:price => "#{rand(1000)}.#{rand(99)}".to_f
|
55
|
+
}
|
56
|
+
products << product
|
57
|
+
end
|
58
|
+
|
59
|
+
Product.import(products)
|
60
|
+
Product.count.should eq(100)
|
61
|
+
end
|
62
|
+
|
63
|
+
it 'should import hash data - out of order' do
|
64
|
+
Product.delete_all
|
65
|
+
products = []
|
66
|
+
1.upto(100) do
|
67
|
+
product = {
|
68
|
+
:vendor => Faker::Name.name,
|
69
|
+
:name => Faker::Name.name,
|
70
|
+
:price => "#{rand(1000)}.#{rand(99)}".to_f,
|
71
|
+
:code => Faker::Lorem.word
|
72
|
+
}
|
73
|
+
products << product
|
74
|
+
end
|
75
|
+
|
76
|
+
Product.import(products)
|
77
|
+
Product.count.should eq(100)
|
78
|
+
end
|
79
|
+
|
80
|
+
|
45
81
|
after(:all) do
|
46
82
|
CreateProducts.down
|
47
83
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: activerecord-jdbc-import
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-05-
|
12
|
+
date: 2013-05-28 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|