activerecord-jdbc-import 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md CHANGED
@@ -9,6 +9,9 @@ and we duck type, so we just want to say 'bulk load this chunk of data'.
9
9
 
10
10
  This library aims to make loading data fast and easy. Just call a method, and it loads your data.
11
11
 
12
+ Note: The code here works, but it is pretty ugly. I am still working
13
+ through the implementation. This is a playground right now.
14
+
12
15
  ## License
13
16
 
14
17
  MIT. Do what you want. If you make changes please contribute them back.
@@ -46,12 +49,48 @@ Or install it yourself as:
46
49
  Product.import(products)
47
50
 
48
51
  Product.count.should eq(100)
52
+
53
+ Or, you might want to use plain hashes. Creating an ActiveRecord object
54
+ for each row might be too much overhead for you:
55
+
56
+ require 'active_record/jdbc/import'
57
+
58
+ class Product < ActiveRecord::Base
59
+ include ActiveRecord::Jdbc::Import
60
+ end
61
+
62
+ products = []
49
63
 
64
+ 1.upto(100) do
65
+ product = {
66
+ :name => "foobar123"
67
+ }
68
+ products << product
69
+ end
70
+
71
+ Product.import(products)
72
+
73
+ Product.count.should eq(100)
74
+
50
75
  It is easy to use, and probably could be easier still. Feel free to fork the code,
51
76
  make changes, fix bugs, etc.
52
77
 
78
+ ## Tricks
79
+
80
+ If you are using MySQL, you can speed up the import by adding the
81
+ following options to your database.yml file:
82
+
83
+ development:
84
+ adapter: mysql
85
+ ...
86
+ options:
87
+ useServerPrepStmts: 'false'
88
+ rewriteBatchedStatements: 'true'
89
+
53
90
  ## TODO
54
91
 
92
+ * Support more column types. Right now, this gem expects text and
93
+ numeric fields.
55
94
  * The `id` column is currently ignored. The library assumes that `id` is going to be autoincremented. Make this optional.
56
95
  * Test with more databases. Right now, Teradata, MySQL, and SQLite3 are all working.
57
96
 
@@ -10,41 +10,103 @@ module ActiveRecord
10
10
 
11
11
  end
12
12
 
13
+ # Code here is ugly.
14
+ # Going to clean up as soon as the whole idea is worked out.
15
+
13
16
  module ClassMethods
17
+
18
+ def import_infile(models)
19
+ return unless models.size > 0
20
+
21
+ start = Time.now
22
+
23
+ CSV.open("/tmp/upload.csv", "w") do |row|
24
+
25
+ row << models.first.keys.map { |a| a.to_s }
26
+
27
+ models.each do |model|
28
+ row << model.values
29
+ end
30
+ end
31
+
32
+ sql = []
33
+ sql << "LOAD DATA INFILE '/tmp/upload.csv'"
34
+ sql << "INTO TABLE #{self.table_name}"
35
+ sql << "FIELDS TERMINATED BY ','"
36
+ sql << "OPTIONALLY ENCLOSED BY '\"'"
37
+ sql << "LINES TERMINATED BY '\\n'"
38
+ sql << "IGNORE 1 LINES"
39
+ sql << "(#{models.first.keys.map { |a| a.to_s }.join(",")});"
40
+
41
+ self.connection.execute(sql.join(" "))
42
+
43
+ p "Wrote tempfile: #{Time.now - start}"
44
+
45
+ end
46
+
14
47
  def import(models)
15
48
  return unless models.size > 0
16
49
 
17
50
  conn = self.connection.jdbc_connection
18
51
 
19
- insert_sql = models.first.to_prepared_sql
52
+ o = self.new
53
+ insert_sql = o.to_prepared_sql
54
+ ordered_columns = o.ordered_columns
20
55
 
21
56
  pstmt = conn.prepareStatement(insert_sql)
22
57
 
23
58
  conn.setAutoCommit(false)
24
59
 
25
- models.each do |model|
26
- i = 1
27
- model.attributes.each_pair do |key, value|
28
-
29
- next if key.to_s == 'id'
30
-
31
- model_type = self.columns_hash[key].type
32
- if model_type == :integer and value.nil?
33
- pstmt.setInt(i, nil)
34
- elsif model_type == :integer
35
- pstmt.setInt(i, value.to_i)
36
- elsif key.to_s.downcase == 'week_date'
37
- pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
38
- elsif model_type == :string and value.nil?
39
- pstmt.setString(i, nil)
40
- else
41
- pstmt.setString(i, value.to_s)
60
+ if models.first.is_a? Hash
61
+ models.each do |model|
62
+ i = 1
63
+ ordered_columns.each do |key|
64
+ next if key.to_s == 'id'
65
+ next if key.to_s == 'created_at'
66
+ next if key.to_s == 'updated_at'
67
+
68
+ column_type = self.columns_hash[key.to_s].type
69
+ value = model[key.to_sym]
70
+
71
+ if column_type == :integer and value.nil?
72
+ pstmt.setInt(i, nil)
73
+ elsif column_type == :integer
74
+ pstmt.setInt(i, value.to_i)
75
+ elsif key.to_s.downcase == 'week_date'
76
+ pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
77
+ elsif column_type == :string and value.nil?
78
+ pstmt.setString(i, nil)
79
+ else
80
+ pstmt.setString(i, value.to_s)
81
+ end
82
+ i += 1
42
83
  end
43
-
44
- i += 1
84
+ pstmt.addBatch()
85
+ end
86
+ else
87
+ models.each do |model|
88
+ i = 1
89
+ model.attributes.each_pair do |key, value|
90
+ next if key.to_s == 'id'
91
+ next if key.to_s == 'created_at'
92
+ next if key.to_s == 'updated_at'
93
+
94
+ column_type = self.columns_hash[key.to_s].type
95
+ if column_type == :integer and value.nil?
96
+ pstmt.setInt(i, nil)
97
+ elsif column_type == :integer
98
+ pstmt.setInt(i, value.to_i)
99
+ elsif key.to_s.downcase == 'week_date'
100
+ pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
101
+ elsif column_type == :string and value.nil?
102
+ pstmt.setString(i, nil)
103
+ else
104
+ pstmt.setString(i, value.to_s)
105
+ end
106
+ i += 1
107
+ end
108
+ pstmt.addBatch()
45
109
  end
46
-
47
- pstmt.addBatch()
48
110
  end
49
111
 
50
112
  pstmt.executeBatch()
@@ -55,23 +117,35 @@ module ActiveRecord
55
117
  end
56
118
  end
57
119
 
120
+ def ordered_columns
121
+ @ordered_columns
122
+ end
123
+
58
124
  def to_prepared_sql
59
125
  conn = self.connection
60
-
126
+
127
+ at_date = DateTime.now
128
+
61
129
  quoted_columns = []
62
130
  quoted_values = []
131
+ @ordered_columns = []
63
132
  attributes_with_values = self.send(:arel_attributes_values, true, true)
64
133
  attributes_with_values.each_pair do |key,value|
65
134
  next if key.name.to_s == 'id'
66
135
  quoted_columns << conn.quote_column_name(key.name)
67
- quoted_values << '?'
136
+ if key.name.to_s == 'created_at' or key.name.to_s == 'updated_at'
137
+ quoted_values << "'#{at_date.to_s(:db)}'"
138
+ else
139
+ @ordered_columns << key.name.to_s
140
+ quoted_values << '?'
141
+ end
68
142
  end
69
-
143
+
70
144
  "INSERT INTO #{self.class.quoted_table_name} " +
71
145
  "(#{quoted_columns.join(', ')}) " +
72
- "VALUES (#{quoted_values.join(', ')});"
146
+ "VALUES (#{quoted_values.join(', ')})"
73
147
  end
74
-
148
+
75
149
  end
76
150
  end
77
151
  end
@@ -1,7 +1,7 @@
1
1
  module Activerecord
2
2
  module Jdbc
3
3
  module Import
4
- VERSION = "0.0.2"
4
+ VERSION = "0.0.3"
5
5
  end
6
6
  end
7
7
  end
data/spec/import_spec.rb CHANGED
@@ -27,6 +27,7 @@ describe ActiveRecord::Jdbc::Import do
27
27
  end
28
28
 
29
29
  it 'should import 100 rows of data' do
30
+ Product.delete_all
30
31
  products = []
31
32
  1.upto(100) do
32
33
  product = Product.new
@@ -42,6 +43,41 @@ describe ActiveRecord::Jdbc::Import do
42
43
 
43
44
  end
44
45
 
46
+ it 'should import hash data' do
47
+ Product.delete_all
48
+ products = []
49
+ 1.upto(100) do
50
+ product = {
51
+ :code => Faker::Lorem.word,
52
+ :name => Faker::Name.name,
53
+ :vendor => Faker::Name.name,
54
+ :price => "#{rand(1000)}.#{rand(99)}".to_f
55
+ }
56
+ products << product
57
+ end
58
+
59
+ Product.import(products)
60
+ Product.count.should eq(100)
61
+ end
62
+
63
+ it 'should import hash data - out of order' do
64
+ Product.delete_all
65
+ products = []
66
+ 1.upto(100) do
67
+ product = {
68
+ :vendor => Faker::Name.name,
69
+ :name => Faker::Name.name,
70
+ :price => "#{rand(1000)}.#{rand(99)}".to_f,
71
+ :code => Faker::Lorem.word
72
+ }
73
+ products << product
74
+ end
75
+
76
+ Product.import(products)
77
+ Product.count.should eq(100)
78
+ end
79
+
80
+
45
81
  after(:all) do
46
82
  CreateProducts.down
47
83
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: activerecord-jdbc-import
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-05-20 00:00:00.000000000 Z
12
+ date: 2013-05-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: activesupport