postgres_upsert 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +11 -0
- data/lib/postgres_upsert/active_record.rb +16 -16
- data/postgres_upsert.gemspec +1 -1
- data/spec/fixtures/no_id.csv +2 -0
- data/spec/pg_upsert_csv_spec.rb +23 -0
- metadata +3 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a30c825147ef323c1e0d60d9fc08e9668b6bdf02
|
4
|
+
data.tar.gz: d3f8799b61a273b4abde1eae7a6b38eea3ba614c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 07649b13cae7be995e12c02e5418bde744f932127587141a160bd09f502f9342f2ce5f2038d20953eed9425043f67c056dde96dec78ed5a8fcf5ec0c11392bea
|
7
|
+
data.tar.gz: e963592a3c91c61d206d018cbcc63aa07995a50165c97dbb88e9352e8b16c76cdcbaef85a9c9b655716d0e0ab387c9c155bf1890200374ce074e5405899929fd
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -75,6 +75,17 @@ This merge/upsert happend in 5 steps (assume your data table is called "users")
|
|
75
75
|
* issue a query to update all records in users with the data in users_temp_### (matching on primary key)
|
76
76
|
* drop the temp table.
|
77
77
|
|
78
|
+
### overriding the key_column
|
79
|
+
|
80
|
+
By default pg_upsert uses the primary key on your ActiveRecord table to determine if each record should be inserted or updated. You can override the column using the :key_field option:
|
81
|
+
|
82
|
+
```ruby
|
83
|
+
User.pg_upsert "/tmp/users.dat", :format => :binary, :key_column => ["external_twitter_id"]
|
84
|
+
```
|
85
|
+
|
86
|
+
obviously, the field you pass must be a unique key in your database (this is not enforced at the moment, but will be)
|
87
|
+
|
88
|
+
|
78
89
|
## Note on Patches/Pull Requests
|
79
90
|
|
80
91
|
* Fork the project
|
@@ -7,7 +7,7 @@ module ActiveRecord
|
|
7
7
|
# * You can map fields from the file to different fields in the table using a map in the options hash
|
8
8
|
# * For further details on usage take a look at the README.md
|
9
9
|
def self.pg_upsert path_or_io, options = {}
|
10
|
-
options.reverse_merge!({:delimiter => ",", :format => :csv, :header => true})
|
10
|
+
options.reverse_merge!({:delimiter => ",", :format => :csv, :header => true, :key_column => primary_key})
|
11
11
|
options_string = options[:format] == :binary ? "BINARY" : "DELIMITER '#{options[:delimiter]}' CSV"
|
12
12
|
|
13
13
|
io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
|
@@ -20,7 +20,7 @@ module ActiveRecord
|
|
20
20
|
destination_table = get_table_name(options)
|
21
21
|
|
22
22
|
columns_string = columns_string_for_copy(columns_list)
|
23
|
-
create_temp_table(copy_table, destination_table, columns_list) if destination_table
|
23
|
+
create_temp_table(copy_table, destination_table, columns_list, options) if destination_table
|
24
24
|
|
25
25
|
connection.raw_connection.copy_data %{COPY #{copy_table} #{columns_string} FROM STDIN #{options_string}} do
|
26
26
|
if block_given?
|
@@ -33,7 +33,7 @@ module ActiveRecord
|
|
33
33
|
end
|
34
34
|
|
35
35
|
if destination_table
|
36
|
-
upsert_from_temp_table(copy_table, destination_table, columns_list)
|
36
|
+
upsert_from_temp_table(copy_table, destination_table, columns_list, options)
|
37
37
|
drop_temp_table(copy_table)
|
38
38
|
end
|
39
39
|
end
|
@@ -80,9 +80,9 @@ module ActiveRecord
|
|
80
80
|
str
|
81
81
|
end
|
82
82
|
|
83
|
-
def self.select_string_for_create(columns_list)
|
83
|
+
def self.select_string_for_create(columns_list, options)
|
84
84
|
columns = columns_list.map(&:to_sym)
|
85
|
-
columns <<
|
85
|
+
columns << options[:key_column].to_sym unless columns.include?(options[:key_column].to_sym)
|
86
86
|
get_columns_string(columns)
|
87
87
|
end
|
88
88
|
|
@@ -119,18 +119,18 @@ module ActiveRecord
|
|
119
119
|
end
|
120
120
|
end
|
121
121
|
|
122
|
-
def self.upsert_from_temp_table(temp_table, dest_table, columns_list)
|
123
|
-
update_from_temp_table(temp_table, dest_table, columns_list)
|
124
|
-
insert_from_temp_table(temp_table, dest_table, columns_list)
|
122
|
+
def self.upsert_from_temp_table(temp_table, dest_table, columns_list, options)
|
123
|
+
update_from_temp_table(temp_table, dest_table, columns_list, options)
|
124
|
+
insert_from_temp_table(temp_table, dest_table, columns_list, options)
|
125
125
|
end
|
126
126
|
|
127
|
-
def self.update_from_temp_table(temp_table, dest_table, columns_list)
|
127
|
+
def self.update_from_temp_table(temp_table, dest_table, columns_list, options)
|
128
128
|
ActiveRecord::Base.connection.execute <<-SQL
|
129
129
|
UPDATE #{dest_table} AS d
|
130
130
|
#{update_set_clause(columns_list)}
|
131
131
|
FROM #{temp_table} as t
|
132
|
-
WHERE t.#{
|
133
|
-
AND d.#{
|
132
|
+
WHERE t.#{options[:key_column]} = d.#{options[:key_column]}
|
133
|
+
AND d.#{options[:key_column]} IS NOT NULL;
|
134
134
|
SQL
|
135
135
|
end
|
136
136
|
|
@@ -142,7 +142,7 @@ module ActiveRecord
|
|
142
142
|
"SET #{command.join(',')}"
|
143
143
|
end
|
144
144
|
|
145
|
-
def self.insert_from_temp_table(temp_table, dest_table, columns_list)
|
145
|
+
def self.insert_from_temp_table(temp_table, dest_table, columns_list, options)
|
146
146
|
columns_string = columns_string_for_insert(columns_list)
|
147
147
|
select_string = select_string_for_insert(columns_list)
|
148
148
|
ActiveRecord::Base.connection.execute <<-SQL
|
@@ -152,13 +152,13 @@ module ActiveRecord
|
|
152
152
|
WHERE NOT EXISTS
|
153
153
|
(SELECT 1
|
154
154
|
FROM #{dest_table} as d
|
155
|
-
WHERE d.#{
|
156
|
-
AND t.#{
|
155
|
+
WHERE d.#{options[:key_column]} = t.#{options[:key_column]})
|
156
|
+
AND t.#{options[:key_column]} IS NOT NULL;
|
157
157
|
SQL
|
158
158
|
end
|
159
159
|
|
160
|
-
def self.create_temp_table(temp_table, dest_table, columns_list)
|
161
|
-
columns_string = select_string_for_create(columns_list)
|
160
|
+
def self.create_temp_table(temp_table, dest_table, columns_list, options)
|
161
|
+
columns_string = select_string_for_create(columns_list, options)
|
162
162
|
ActiveRecord::Base.connection.execute <<-SQL
|
163
163
|
SET client_min_messages=WARNING;
|
164
164
|
DROP TABLE IF EXISTS #{temp_table};
|
data/postgres_upsert.gemspec
CHANGED
data/spec/pg_upsert_csv_spec.rb
CHANGED
@@ -4,6 +4,7 @@ describe "pg_upsert from file with CSV format" do
|
|
4
4
|
before(:each) do
|
5
5
|
ActiveRecord::Base.connection.execute %{
|
6
6
|
TRUNCATE TABLE test_models;
|
7
|
+
TRUNCATE TABLE three_columns;
|
7
8
|
SELECT setval('test_models_id_seq', 1, false);
|
8
9
|
}
|
9
10
|
end
|
@@ -183,5 +184,27 @@ describe "pg_upsert from file with CSV format" do
|
|
183
184
|
).to eq('id' => 1, 'data' => 'test data 1', 'extra' => "neva change!", 'created_at' => original_created_at, 'updated_at' => timestamp)
|
184
185
|
end
|
185
186
|
end
|
187
|
+
|
188
|
+
context 'overriding the comparison column' do
|
189
|
+
it 'updates records based the match column option if its passed in' do
|
190
|
+
three_col = ThreeColumn.create(id: 1, data: "old stuff", extra: "neva change!")
|
191
|
+
file = File.open(File.expand_path('spec/fixtures/no_id.csv'), 'r')
|
192
|
+
|
193
|
+
|
194
|
+
ThreeColumn.pg_upsert(file, :key_column => "data")
|
195
|
+
expect(
|
196
|
+
three_col.reload.extra
|
197
|
+
).to eq("ABC: Always Be Changing.")
|
198
|
+
end
|
199
|
+
|
200
|
+
it 'inserts records if the passed match column doesnt exist' do
|
201
|
+
file = File.open(File.expand_path('spec/fixtures/no_id.csv'), 'r')
|
202
|
+
|
203
|
+
ThreeColumn.pg_upsert(file, :key_column => "data")
|
204
|
+
expect(
|
205
|
+
ThreeColumn.last.attributes
|
206
|
+
).to include("id" => 1, "data" => "old stuff", "extra" => "ABC: Always Be Changing.")
|
207
|
+
end
|
208
|
+
end
|
186
209
|
end
|
187
210
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: postgres_upsert
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Steve Mitchell
|
@@ -130,6 +130,7 @@ files:
|
|
130
130
|
- spec/fixtures/comma_with_header_and_comma_values.csv
|
131
131
|
- spec/fixtures/comma_with_header_and_unquoted_comma.csv
|
132
132
|
- spec/fixtures/comma_without_header.csv
|
133
|
+
- spec/fixtures/no_id.csv
|
133
134
|
- spec/fixtures/reserved_word_model.rb
|
134
135
|
- spec/fixtures/reserved_words.csv
|
135
136
|
- spec/fixtures/semicolon_with_different_header.csv
|
@@ -176,6 +177,7 @@ test_files:
|
|
176
177
|
- spec/fixtures/comma_with_header_and_comma_values.csv
|
177
178
|
- spec/fixtures/comma_with_header_and_unquoted_comma.csv
|
178
179
|
- spec/fixtures/comma_without_header.csv
|
180
|
+
- spec/fixtures/no_id.csv
|
179
181
|
- spec/fixtures/reserved_word_model.rb
|
180
182
|
- spec/fixtures/reserved_words.csv
|
181
183
|
- spec/fixtures/semicolon_with_different_header.csv
|