activerecord-jdbc-import 0.0.2 → 0.0.3

Sign up to get free protection for your applications and to get access to all the features.
data/README.md CHANGED
@@ -9,6 +9,9 @@ and we duck type, so we just want to say 'bulk load this chunk of data'.
9
9
 
10
10
  This library aims to make loading data fast and easy. Just call a method, and it loads your data.
11
11
 
12
+ Note: The code here works, but it is pretty ugly. I am still working
13
+ through the implementation. This is a playground right now.
14
+
12
15
  ## License
13
16
 
14
17
  MIT. Do what you want. If you make changes please contribute them back.
@@ -46,12 +49,48 @@ Or install it yourself as:
46
49
  Product.import(products)
47
50
 
48
51
  Product.count.should eq(100)
52
+
53
+ Or, you might want to use plain hashes. Creating an ActiveRecord object
54
+ for each row might be too much overhead for you:
55
+
56
+ require 'active_record/jdbc/import'
57
+
58
+ class Product < ActiveRecord::Base
59
+ include ActiveRecord::Jdbc::Import
60
+ end
61
+
62
+ products = []
49
63
 
64
+ 1.upto(100) do
65
+ product = {
66
+ :name => "foobar123"
67
+ }
68
+ products << product
69
+ end
70
+
71
+ Product.import(products)
72
+
73
+ Product.count.should eq(100)
74
+
50
75
  It is easy to use, and probably could be easier still. Feel free to fork the code,
51
76
  make changes, fix bugs, etc.
52
77
 
78
+ ## Tricks
79
+
80
+ If you are using MySQL, you can speed up the import by adding the
81
+ following options to your database.yml file:
82
+
83
+ development:
84
+ adapter: mysql
85
+ ...
86
+ options:
87
+ useServerPrepStmts: 'false'
88
+ rewriteBatchedStatements: 'true'
89
+
53
90
  ## TODO
54
91
 
92
+ * Support more column types. Right now, this gem expects text and
93
+ numeric fields.
55
94
  * The `id` column is currently ignored. The library assumes that `id` is going to be autoincremented. Make this optional.
56
95
  * Test with more databases. Right now, Teradata, MySQL, and SQLite3 are all working.
57
96
 
@@ -10,41 +10,103 @@ module ActiveRecord
10
10
 
11
11
  end
12
12
 
13
+ # Code here is ugly.
14
+ # Going to clean up as soon as the whole idea is worked out.
15
+
13
16
  module ClassMethods
17
+
18
+ def import_infile(models)
19
+ return unless models.size > 0
20
+
21
+ start = Time.now
22
+
23
+ CSV.open("/tmp/upload.csv", "w") do |row|
24
+
25
+ row << models.first.keys.map { |a| a.to_s }
26
+
27
+ models.each do |model|
28
+ row << model.values
29
+ end
30
+ end
31
+
32
+ sql = []
33
+ sql << "LOAD DATA INFILE '/tmp/upload.csv'"
34
+ sql << "INTO TABLE #{self.table_name}"
35
+ sql << "FIELDS TERMINATED BY ','"
36
+ sql << "OPTIONALLY ENCLOSED BY '\"'"
37
+ sql << "LINES TERMINATED BY '\\n'"
38
+ sql << "IGNORE 1 LINES"
39
+ sql << "(#{models.first.keys.map { |a| a.to_s }.join(",")});"
40
+
41
+ self.connection.execute(sql.join(" "))
42
+
43
+ p "Wrote tempfile: #{Time.now - start}"
44
+
45
+ end
46
+
14
47
  def import(models)
15
48
  return unless models.size > 0
16
49
 
17
50
  conn = self.connection.jdbc_connection
18
51
 
19
- insert_sql = models.first.to_prepared_sql
52
+ o = self.new
53
+ insert_sql = o.to_prepared_sql
54
+ ordered_columns = o.ordered_columns
20
55
 
21
56
  pstmt = conn.prepareStatement(insert_sql)
22
57
 
23
58
  conn.setAutoCommit(false)
24
59
 
25
- models.each do |model|
26
- i = 1
27
- model.attributes.each_pair do |key, value|
28
-
29
- next if key.to_s == 'id'
30
-
31
- model_type = self.columns_hash[key].type
32
- if model_type == :integer and value.nil?
33
- pstmt.setInt(i, nil)
34
- elsif model_type == :integer
35
- pstmt.setInt(i, value.to_i)
36
- elsif key.to_s.downcase == 'week_date'
37
- pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
38
- elsif model_type == :string and value.nil?
39
- pstmt.setString(i, nil)
40
- else
41
- pstmt.setString(i, value.to_s)
60
+ if models.first.is_a? Hash
61
+ models.each do |model|
62
+ i = 1
63
+ ordered_columns.each do |key|
64
+ next if key.to_s == 'id'
65
+ next if key.to_s == 'created_at'
66
+ next if key.to_s == 'updated_at'
67
+
68
+ column_type = self.columns_hash[key.to_s].type
69
+ value = model[key.to_sym]
70
+
71
+ if column_type == :integer and value.nil?
72
+ pstmt.setInt(i, nil)
73
+ elsif column_type == :integer
74
+ pstmt.setInt(i, value.to_i)
75
+ elsif key.to_s.downcase == 'week_date'
76
+ pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
77
+ elsif column_type == :string and value.nil?
78
+ pstmt.setString(i, nil)
79
+ else
80
+ pstmt.setString(i, value.to_s)
81
+ end
82
+ i += 1
42
83
  end
43
-
44
- i += 1
84
+ pstmt.addBatch()
85
+ end
86
+ else
87
+ models.each do |model|
88
+ i = 1
89
+ model.attributes.each_pair do |key, value|
90
+ next if key.to_s == 'id'
91
+ next if key.to_s == 'created_at'
92
+ next if key.to_s == 'updated_at'
93
+
94
+ column_type = self.columns_hash[key.to_s].type
95
+ if column_type == :integer and value.nil?
96
+ pstmt.setInt(i, nil)
97
+ elsif column_type == :integer
98
+ pstmt.setInt(i, value.to_i)
99
+ elsif key.to_s.downcase == 'week_date'
100
+ pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
101
+ elsif column_type == :string and value.nil?
102
+ pstmt.setString(i, nil)
103
+ else
104
+ pstmt.setString(i, value.to_s)
105
+ end
106
+ i += 1
107
+ end
108
+ pstmt.addBatch()
45
109
  end
46
-
47
- pstmt.addBatch()
48
110
  end
49
111
 
50
112
  pstmt.executeBatch()
@@ -55,23 +117,35 @@ module ActiveRecord
55
117
  end
56
118
  end
57
119
 
120
+ def ordered_columns
121
+ @ordered_columns
122
+ end
123
+
58
124
  def to_prepared_sql
59
125
  conn = self.connection
60
-
126
+
127
+ at_date = DateTime.now
128
+
61
129
  quoted_columns = []
62
130
  quoted_values = []
131
+ @ordered_columns = []
63
132
  attributes_with_values = self.send(:arel_attributes_values, true, true)
64
133
  attributes_with_values.each_pair do |key,value|
65
134
  next if key.name.to_s == 'id'
66
135
  quoted_columns << conn.quote_column_name(key.name)
67
- quoted_values << '?'
136
+ if key.name.to_s == 'created_at' or key.name.to_s == 'updated_at'
137
+ quoted_values << "'#{at_date.to_s(:db)}'"
138
+ else
139
+ @ordered_columns << key.name.to_s
140
+ quoted_values << '?'
141
+ end
68
142
  end
69
-
143
+
70
144
  "INSERT INTO #{self.class.quoted_table_name} " +
71
145
  "(#{quoted_columns.join(', ')}) " +
72
- "VALUES (#{quoted_values.join(', ')});"
146
+ "VALUES (#{quoted_values.join(', ')})"
73
147
  end
74
-
148
+
75
149
  end
76
150
  end
77
151
  end
@@ -1,7 +1,7 @@
1
1
  module Activerecord
2
2
  module Jdbc
3
3
  module Import
4
- VERSION = "0.0.2"
4
+ VERSION = "0.0.3"
5
5
  end
6
6
  end
7
7
  end
data/spec/import_spec.rb CHANGED
@@ -27,6 +27,7 @@ describe ActiveRecord::Jdbc::Import do
27
27
  end
28
28
 
29
29
  it 'should import 100 rows of data' do
30
+ Product.delete_all
30
31
  products = []
31
32
  1.upto(100) do
32
33
  product = Product.new
@@ -42,6 +43,41 @@ describe ActiveRecord::Jdbc::Import do
42
43
 
43
44
  end
44
45
 
46
+ it 'should import hash data' do
47
+ Product.delete_all
48
+ products = []
49
+ 1.upto(100) do
50
+ product = {
51
+ :code => Faker::Lorem.word,
52
+ :name => Faker::Name.name,
53
+ :vendor => Faker::Name.name,
54
+ :price => "#{rand(1000)}.#{rand(99)}".to_f
55
+ }
56
+ products << product
57
+ end
58
+
59
+ Product.import(products)
60
+ Product.count.should eq(100)
61
+ end
62
+
63
+ it 'should import hash data - out of order' do
64
+ Product.delete_all
65
+ products = []
66
+ 1.upto(100) do
67
+ product = {
68
+ :vendor => Faker::Name.name,
69
+ :name => Faker::Name.name,
70
+ :price => "#{rand(1000)}.#{rand(99)}".to_f,
71
+ :code => Faker::Lorem.word
72
+ }
73
+ products << product
74
+ end
75
+
76
+ Product.import(products)
77
+ Product.count.should eq(100)
78
+ end
79
+
80
+
45
81
  after(:all) do
46
82
  CreateProducts.down
47
83
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: activerecord-jdbc-import
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-05-20 00:00:00.000000000 Z
12
+ date: 2013-05-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: activesupport