upsert 2.9.10-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/.ruby-version +1 -0
- data/.standard.yml +1 -0
- data/.travis.yml +63 -0
- data/.yardopts +2 -0
- data/CHANGELOG +265 -0
- data/Gemfile +20 -0
- data/LICENSE +24 -0
- data/README.md +411 -0
- data/Rakefile +54 -0
- data/lib/upsert.rb +284 -0
- data/lib/upsert/active_record_upsert.rb +12 -0
- data/lib/upsert/binary.rb +8 -0
- data/lib/upsert/column_definition.rb +79 -0
- data/lib/upsert/column_definition/mysql.rb +24 -0
- data/lib/upsert/column_definition/postgresql.rb +66 -0
- data/lib/upsert/column_definition/sqlite3.rb +34 -0
- data/lib/upsert/connection.rb +37 -0
- data/lib/upsert/connection/Java_ComMysqlJdbc_JDBC4Connection.rb +31 -0
- data/lib/upsert/connection/Java_OrgPostgresqlJdbc_PgConnection.rb +33 -0
- data/lib/upsert/connection/Java_OrgSqlite_Conn.rb +17 -0
- data/lib/upsert/connection/Mysql2_Client.rb +76 -0
- data/lib/upsert/connection/PG_Connection.rb +35 -0
- data/lib/upsert/connection/SQLite3_Database.rb +28 -0
- data/lib/upsert/connection/jdbc.rb +105 -0
- data/lib/upsert/connection/postgresql.rb +24 -0
- data/lib/upsert/connection/sqlite3.rb +19 -0
- data/lib/upsert/merge_function.rb +73 -0
- data/lib/upsert/merge_function/Java_ComMysqlJdbc_JDBC4Connection.rb +42 -0
- data/lib/upsert/merge_function/Java_OrgPostgresqlJdbc_PgConnection.rb +27 -0
- data/lib/upsert/merge_function/Java_OrgSqlite_Conn.rb +10 -0
- data/lib/upsert/merge_function/Mysql2_Client.rb +36 -0
- data/lib/upsert/merge_function/PG_Connection.rb +26 -0
- data/lib/upsert/merge_function/SQLite3_Database.rb +10 -0
- data/lib/upsert/merge_function/mysql.rb +66 -0
- data/lib/upsert/merge_function/postgresql.rb +365 -0
- data/lib/upsert/merge_function/sqlite3.rb +43 -0
- data/lib/upsert/row.rb +59 -0
- data/lib/upsert/version.rb +3 -0
- data/spec/active_record_upsert_spec.rb +26 -0
- data/spec/binary_spec.rb +21 -0
- data/spec/correctness_spec.rb +190 -0
- data/spec/database_functions_spec.rb +106 -0
- data/spec/database_spec.rb +121 -0
- data/spec/hstore_spec.rb +249 -0
- data/spec/jruby_spec.rb +9 -0
- data/spec/logger_spec.rb +52 -0
- data/spec/misc/get_postgres_reserved_words.rb +12 -0
- data/spec/misc/mysql_reserved.txt +226 -0
- data/spec/misc/pg_reserved.txt +742 -0
- data/spec/multibyte_spec.rb +27 -0
- data/spec/postgresql_spec.rb +94 -0
- data/spec/precision_spec.rb +11 -0
- data/spec/reserved_words_spec.rb +50 -0
- data/spec/sequel_spec.rb +57 -0
- data/spec/spec_helper.rb +417 -0
- data/spec/speed_spec.rb +44 -0
- data/spec/threaded_spec.rb +57 -0
- data/spec/timezones_spec.rb +58 -0
- data/spec/type_safety_spec.rb +12 -0
- data/travis/install_postgres.sh +18 -0
- data/travis/run_docker_db.sh +20 -0
- data/travis/tune_mysql.sh +7 -0
- data/upsert-java.gemspec +14 -0
- data/upsert.gemspec +13 -0
- data/upsert.gemspec.common +106 -0
- metadata +373 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 7eb840a86c1973e535fae818ad6212aeae072512
|
4
|
+
data.tar.gz: 617afaddc0e769d688a4e3dba95e9beeef436596
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 900bce378101ac8da7235eb365f9a1b568b5d95480c9a32193186aaa34293b7ea1efb2af6557c833604646b9bd1b052bc3bc80bc384497e627b24039666078f9
|
7
|
+
data.tar.gz: 5d48a452e7d60bab795739152ddae77f169fca3685d21b3b931bf0f73998ac1af37ce9332d68a885873e2d07cf198125369b721dcae54ac48c11dd4d6edfe9e2
|
data/.gitignore
ADDED
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
jruby-9.1.17.0
|
data/.standard.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby_version: 2.2.4
|
data/.travis.yml
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
dist: xenial
|
2
|
+
language: ruby
|
3
|
+
cache: bundler
|
4
|
+
services:
|
5
|
+
- docker
|
6
|
+
rvm:
|
7
|
+
- 2.6
|
8
|
+
- 2.5
|
9
|
+
- 2.4
|
10
|
+
- 2.3
|
11
|
+
- 2.2
|
12
|
+
- jruby-9.1.14.0
|
13
|
+
- jruby-9.1.17.0
|
14
|
+
- jruby-9.2.7.0
|
15
|
+
env:
|
16
|
+
global:
|
17
|
+
- USERNAME=travis
|
18
|
+
- PASSWORD=
|
19
|
+
- DB_USER=upsert_test
|
20
|
+
- DB_PASSWORD=upsert_test
|
21
|
+
- DB_NAME=upsert_test
|
22
|
+
matrix:
|
23
|
+
- DB=postgresql DB_VERSION=postgres:9.4
|
24
|
+
- DB=postgresql DB_VERSION=postgres:9.5
|
25
|
+
- DB=postgresql DB_VERSION=postgres:9.6
|
26
|
+
- DB=postgresql DB_VERSION=postgres:10
|
27
|
+
- DB=postgresql DB_VERSION=postgres:11
|
28
|
+
- DB=postgresql DB_VERSION=postgres:12
|
29
|
+
- DB=postgresql DB_VERSION=postgres:9.4 UNIQUE_CONSTRAINT=true
|
30
|
+
- DB=postgresql DB_VERSION=postgres:9.5 UNIQUE_CONSTRAINT=true
|
31
|
+
- DB=postgresql DB_VERSION=postgres:9.6 UNIQUE_CONSTRAINT=true
|
32
|
+
- DB=postgresql DB_VERSION=postgres:10 UNIQUE_CONSTRAINT=true
|
33
|
+
- DB=postgresql DB_VERSION=postgres:11 UNIQUE_CONSTRAINT=true
|
34
|
+
- DB=postgresql DB_VERSION=postgres:12 UNIQUE_CONSTRAINT=true
|
35
|
+
- DB=mysql DB_VERSION=mysql:5.6
|
36
|
+
- DB=mysql DB_VERSION=mysql:5.7
|
37
|
+
- DB=mysql DB_VERSION=mysql:8
|
38
|
+
matrix:
|
39
|
+
exclude:
|
40
|
+
- rvm: 2.6
|
41
|
+
env: DB=postgresql DB_VERSION=postgres:9.4
|
42
|
+
- rvm: 2.6
|
43
|
+
env: DB=postgresql DB_VERSION=postgres:9.5
|
44
|
+
- rvm: 2.6
|
45
|
+
env: DB=postgresql DB_VERSION=postgres:9.4 UNIQUE_CONSTRAINT=true
|
46
|
+
- rvm: 2.6
|
47
|
+
env: DB=postgresql DB_VERSION=postgres:9.5 UNIQUE_CONSTRAINT=true
|
48
|
+
- rvm: jruby-9.2.7
|
49
|
+
env: DB=postgresql DB_VERSION=postgres:9.4
|
50
|
+
- rvm: jruby-9.2.7
|
51
|
+
env: DB=postgresql DB_VERSION=postgres:9.5
|
52
|
+
- rvm: jruby-9.2.7
|
53
|
+
env: DB=postgresql DB_VERSION=postgres:9.4 UNIQUE_CONSTRAINT=true
|
54
|
+
- rvm: jruby-9.2.7
|
55
|
+
env: DB=postgresql DB_VERSION=postgres:9.5 UNIQUE_CONSTRAINT=true
|
56
|
+
allow_failures:
|
57
|
+
- env: DB=postgresql DB_VERSION=postgres:12
|
58
|
+
- env: DB=postgresql DB_VERSION=postgres:12 UNIQUE_CONSTRAINT=true
|
59
|
+
before_install:
|
60
|
+
- ./travis/run_docker_db.sh
|
61
|
+
- bundle --version
|
62
|
+
- gem --version
|
63
|
+
script: ./travis/run_specs.sh
|
data/.yardopts
ADDED
data/CHANGELOG
ADDED
@@ -0,0 +1,265 @@
|
|
1
|
+
-- no version -- / 2019-06-05
|
2
|
+
|
3
|
+
* Enhancements
|
4
|
+
|
5
|
+
* Bump development Ruby version to 2.5 since Ruby 2.2 is no longer supported.
|
6
|
+
This should not affect usage of the gem, only local development for people
|
7
|
+
working *on* the gem. Ruby 2.2 is also not dropped from Upsert compatibility
|
8
|
+
at this time but you should consider upgrading to newer Ruby versions anyway.
|
9
|
+
|
10
|
+
2.2.1 / 2017-04-20
|
11
|
+
|
12
|
+
* Bug fixes
|
13
|
+
|
14
|
+
* Fix unique constraint detection on pg >9.5.5 (@pnomolos https://github.com/seamusabshere/upsert/pull/99)
|
15
|
+
* Fix Ruby 1.9 tests
|
16
|
+
|
17
|
+
2.2.0 / 2017-04-14
|
18
|
+
|
19
|
+
* Enhancements
|
20
|
+
|
21
|
+
* Use native "upsert" on Postgres 9.5+! (thanks to @pnomolos https://github.com/seamusabshere/upsert/pull/79)
|
22
|
+
* More modern CI tests
|
23
|
+
|
24
|
+
2.1.2 / 2016-02-25
|
25
|
+
|
26
|
+
* Enhancements
|
27
|
+
|
28
|
+
* Test on Ruby 2.3 - thanks @Ch4s3 https://github.com/seamusabshere/upsert/pull/70
|
29
|
+
|
30
|
+
* Bug fixes
|
31
|
+
|
32
|
+
* Stop using Thread.exclusive - thanks @hpetru https://github.com/seamusabshere/upsert/pull/67
|
33
|
+
|
34
|
+
2.1.1 / 2016-02-12
|
35
|
+
|
36
|
+
* Enhancements
|
37
|
+
|
38
|
+
* Assume function exists to avoid huge amounts of recreation
|
39
|
+
|
40
|
+
2.1.0 / 2015-03-13
|
41
|
+
|
42
|
+
* Bug fixes
|
43
|
+
|
44
|
+
* Thread safety with Sidekiq! thanks @evadne and @thbar ! https://github.com/seamusabshere/upsert/pull/47
|
45
|
+
|
46
|
+
* Known issues
|
47
|
+
|
48
|
+
* speed_spec fails against activerecord-import on mysql... need some advice on properly testing it
|
49
|
+
|
50
|
+
2.0.4 / 2015-01-27
|
51
|
+
|
52
|
+
* Bug fixes
|
53
|
+
|
54
|
+
* Support mysql returning column info as symbols - thanks @pnomolos
|
55
|
+
|
56
|
+
* Enhancements
|
57
|
+
|
58
|
+
* Travis and misc test fixes - thanks @raviolicode
|
59
|
+
* Backwards compat with 1.8 - thanks @raviolicode
|
60
|
+
|
61
|
+
2.0.3 / 2013-11-27
|
62
|
+
|
63
|
+
* Bug fixes
|
64
|
+
|
65
|
+
* Add parentheses to equality expressions
|
66
|
+
|
67
|
+
* Enhancements
|
68
|
+
|
69
|
+
* On Postgres, use (A = B OR (A IS NULL AND B IS NULL)) instead of A IS NOT DISTINCT FROM B because the latter does not use indexes
|
70
|
+
* pass :eager_nullify => false as the third argument to Upsert#row to disable clearing of null HStore keys
|
71
|
+
|
72
|
+
2.0.2 / 2013-11-06
|
73
|
+
|
74
|
+
* Bug fixes
|
75
|
+
|
76
|
+
* Properly check for NULL equality when creating the UPSERT functions - thanks @pnomolos - https://github.com/seamusabshere/upsert/issues/25
|
77
|
+
* When using Mysql2 client (MRI), don't pass timezone to DateTime columns - thanks @kjeremy! - https://github.com/seamusabshere/upsert/issues/24
|
78
|
+
|
79
|
+
2.0.1 / 2013-07-24
|
80
|
+
|
81
|
+
* Bug fixes
|
82
|
+
|
83
|
+
* Rookie mistake - gsub!'ed an input arg, sorry. Blew up on a frozen string, thank goodness.
|
84
|
+
|
85
|
+
2.0.0 / 2013-07-24
|
86
|
+
|
87
|
+
* Breaking changes
|
88
|
+
|
89
|
+
* For Postgres Hstore, null keys are deliberately deleted - so there's no way to set "foo" => NULL - "foo" will just get deleted as a key
|
90
|
+
* Name merge functions "upsert1_2_0" instead of "upsert_"
|
91
|
+
|
92
|
+
1.2.0 / 2013-04-04
|
93
|
+
|
94
|
+
* Breaking changes
|
95
|
+
|
96
|
+
* columns named "created_at" and "created_on" will only be set if it's a new row - thanks @derekharmel! https://github.com/seamusabshere/upsert/pull/15
|
97
|
+
|
98
|
+
* Enhancements
|
99
|
+
|
100
|
+
* Detect invalid columns passed in either selector or setter - inspired by @atandrau, thanks! https://github.com/seamusabshere/upsert/issues/18
|
101
|
+
|
102
|
+
* Bug fixes
|
103
|
+
|
104
|
+
* Always convert symbols to strings when used as bind vars - thanks @towerhe! - https://github.com/seamusabshere/upsert/pull/16
|
105
|
+
|
106
|
+
1.1.7 / 2013-01-15
|
107
|
+
|
108
|
+
* Enhancements
|
109
|
+
|
110
|
+
* :assume_function_exists option to avoid creating same merge function over and over
|
111
|
+
* Don't die on first occurrence of "tuple concurrently updated"
|
112
|
+
|
113
|
+
1.1.6 / 2012-12-20
|
114
|
+
|
115
|
+
* Bug fixes
|
116
|
+
|
117
|
+
* Require pg-hstore >=1.1.1 which doesn't escape single quotes and backslashes when output is going to be used in a bind variable (as it is here)
|
118
|
+
|
119
|
+
1.1.5 / 2012-12-10
|
120
|
+
|
121
|
+
* Bug fixes
|
122
|
+
|
123
|
+
* Properly cast "timestamp without time zone" in postgres - thanks @markmarijnissen! https://github.com/seamusabshere/upsert/issues/11
|
124
|
+
|
125
|
+
1.1.4 / 2012-12-06
|
126
|
+
|
127
|
+
* Enhancements
|
128
|
+
|
129
|
+
* Use latest pg-hstore gem with standardized namespace
|
130
|
+
|
131
|
+
1.1.3 / 2012-12-06
|
132
|
+
|
133
|
+
* Bug fix
|
134
|
+
|
135
|
+
* Don't die/fail trying to update an HStore column that has reverted to NULL.
|
136
|
+
|
137
|
+
1.1.2 / 2012-12-06
|
138
|
+
|
139
|
+
* Enhancements
|
140
|
+
|
141
|
+
* Support for PostgreSQL's HStore
|
142
|
+
|
143
|
+
1.1.1 / 2012-12-03
|
144
|
+
|
145
|
+
* Bug fixes
|
146
|
+
|
147
|
+
* Removed inadvertent dependency on ActiveSupport - thanks @thbar! https://github.com/seamusabshere/upsert/issues/10
|
148
|
+
|
149
|
+
1.1.0 / 2012-11-26
|
150
|
+
|
151
|
+
* Enhancements
|
152
|
+
|
153
|
+
* Works on JRuby using bare-metal JDBC!
|
154
|
+
* Simplified.
|
155
|
+
|
156
|
+
1.0.2 / 2012-11-12
|
157
|
+
|
158
|
+
* Bug fixes
|
159
|
+
|
160
|
+
* Fix filenames - they were updated on an apparently case-insensitive setup. Thanks @ihough! (https://github.com/seamusabshere/upsert/pull/8)
|
161
|
+
* Deliberately drop MySQL procedures before creating them. Also thanks to @ihough!
|
162
|
+
|
163
|
+
1.0.1 / 2012-11-07
|
164
|
+
|
165
|
+
* Bug fixes
|
166
|
+
|
167
|
+
* Fix incorrect gem description
|
168
|
+
|
169
|
+
1.0.0 / 2012-11-07
|
170
|
+
|
171
|
+
* Breaking changes (well, not really)
|
172
|
+
|
173
|
+
* Not using INSERT ... ON DUPLICATE KEY UPDATE for MySQL!
|
174
|
+
|
175
|
+
* Enhancements
|
176
|
+
|
177
|
+
* Replaced ON DUPLICATE KEY with a true merge function (procedure)
|
178
|
+
* Simplified code - buffering is no longer used anywhere
|
179
|
+
* Clarified documentation
|
180
|
+
|
181
|
+
* Bug fixes
|
182
|
+
|
183
|
+
* MySQL upserts won't fail if you have a multi-key selector and no multi-column UNIQUE index to cover them (https://github.com/seamusabshere/upsert/issues/6)
|
184
|
+
|
185
|
+
0.5.0 / 2012-09-21
|
186
|
+
|
187
|
+
* Breaking changes (well, not really)
|
188
|
+
|
189
|
+
* "document" (as in the second argument to #row) has been renamed to "setter"!
|
190
|
+
|
191
|
+
* Bug fixes
|
192
|
+
|
193
|
+
* If you say upsert({:name => 'Jerry', :color => 'red'}), make sure that it only affects rows really meeting those conditions
|
194
|
+
* Always sort selector and setter keys - i.e., column names - before doing anything with them
|
195
|
+
* Support PostgreSQL 9.1+
|
196
|
+
* Support MRI 1.8
|
197
|
+
|
198
|
+
* Enhancements
|
199
|
+
|
200
|
+
* Slightly faster benchmarks for SQlite3 and MySQL
|
201
|
+
* Slightly slower on PostgreSQL (probably because the merge function requires more arguments), but more accurate
|
202
|
+
* Slightly clearer code structure
|
203
|
+
* Use bind parameters instead of quoting for PostgreSQL and SQLite3.
|
204
|
+
* Provide Upsert.clear_database_functions(connection) (currently only for PostgreSQL)
|
205
|
+
* Don't subclass String for Upset::Binary... hopefully save some strcpy()s?
|
206
|
+
|
207
|
+
0.4.0 / 2012-09-04
|
208
|
+
|
209
|
+
* Bug fixes
|
210
|
+
|
211
|
+
* Don't raise TooBig - rely on Mysql2 to complain about oversized packets
|
212
|
+
|
213
|
+
* Enhancements
|
214
|
+
|
215
|
+
* Re-use PostgreSQL merge functions across connections, even outside of batch mode. Huzzah!
|
216
|
+
* For MySQL, increase speed for one-off upserts by not checking packet size
|
217
|
+
* Allow configuring Upsert.logger. Defaults to Rails.logger or Logger.new($stderr). If you set env var UPSERT_DEBUG=true then it will set log level to debug.
|
218
|
+
|
219
|
+
0.3.4 / 2012-07-03
|
220
|
+
|
221
|
+
* Bug fixes
|
222
|
+
|
223
|
+
* Allow upserting by auto-increment primary key (thanks @atandrau https://github.com/seamusabshere/upsert/issues/3)
|
224
|
+
|
225
|
+
* Enhancements
|
226
|
+
|
227
|
+
* Make setter an optional argument
|
228
|
+
|
229
|
+
0.3.3 / 2012-06-26
|
230
|
+
|
231
|
+
* Bug fixes
|
232
|
+
|
233
|
+
* Properly quote table names - don't assume that everybody has ANSI_QUOTES turned on in MySQL :)
|
234
|
+
|
235
|
+
0.3.2 / 2012-06-22
|
236
|
+
|
237
|
+
* Enhancements
|
238
|
+
|
239
|
+
* Make sure ::PGconn is recognized as ::PG::Connection (thanks @joevandyk https://github.com/seamusabshere/upsert/issues/2)
|
240
|
+
|
241
|
+
0.3.1 / 2012-06-21
|
242
|
+
|
243
|
+
* Bug fixes
|
244
|
+
|
245
|
+
* On PostgreSQL, stop nullifying columns that weren't even involved in the upsert
|
246
|
+
* On SQLite, properly join WHERE conditions with ' AND ' instead of ','
|
247
|
+
|
248
|
+
0.3.0 / 2012-06-21
|
249
|
+
|
250
|
+
* Enhancements
|
251
|
+
|
252
|
+
* Remove all the sampling - just keep a cumulative total of sql bytes as we build up an ON DUPLICATE KEY UPDATE query.
|
253
|
+
* Deprecate Upsert.stream in favor of Upsert.batch (but provide an alias for backwards compat)
|
254
|
+
|
255
|
+
0.2.2 / 2012-06-21
|
256
|
+
|
257
|
+
* Bug fixes
|
258
|
+
|
259
|
+
* Correct and simplify how sql length is calculated when batching MySQL upserts.
|
260
|
+
|
261
|
+
0.2.1 / 2012-06-21
|
262
|
+
|
263
|
+
* Enhancements
|
264
|
+
|
265
|
+
* Added support for Ruby 1.8.7
|
data/Gemfile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
# Specify your gem's dependencies in upsert.gemspec
|
4
|
+
|
5
|
+
if Gem::Version.new(Bundler::VERSION) >= Gem::Version.new("2.0.0")
|
6
|
+
gemspec glob: RUBY_PLATFORM == "java" ? "upsert-java.gemspec" : "upsert.gemspec"
|
7
|
+
else
|
8
|
+
gemspec name: RUBY_PLATFORM == "java" ? "upsert-java" : "upsert"
|
9
|
+
end
|
10
|
+
|
11
|
+
case RUBY_PLATFORM
|
12
|
+
when "java"
|
13
|
+
gem "ffi", platforms: :jruby
|
14
|
+
else
|
15
|
+
gem "ffi"
|
16
|
+
end
|
17
|
+
|
18
|
+
group "test" do
|
19
|
+
gem "testmetrics_rspec"
|
20
|
+
end
|
data/LICENSE
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
Copyright (c) 2013-2019 Seamus Abshere
|
2
|
+
Copyright (c) 2017-2019 Philip Schalm
|
3
|
+
Portions Copyright (c) 2019 The JRuby Team
|
4
|
+
|
5
|
+
MIT License
|
6
|
+
|
7
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
8
|
+
a copy of this software and associated documentation files (the
|
9
|
+
"Software"), to deal in the Software without restriction, including
|
10
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
11
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
12
|
+
permit persons to whom the Software is furnished to do so, subject to
|
13
|
+
the following conditions:
|
14
|
+
|
15
|
+
The above copyright notice and this permission notice shall be
|
16
|
+
included in all copies or substantial portions of the Software.
|
17
|
+
|
18
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
19
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
20
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
21
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
22
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
23
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
24
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,411 @@
|
|
1
|
+
# Upsert
|
2
|
+
|
3
|
+
[![Build Status](https://travis-ci.org/seamusabshere/upsert.svg?branch=master)](https://travis-ci.org/seamusabshere/upsert)
|
4
|
+
|
5
|
+
Make it easy to upsert on traditional RDBMS like MySQL, PostgreSQL, and SQLite3—hey look NoSQL!. Transparently creates (and re-uses) stored procedures/functions when necessary.
|
6
|
+
|
7
|
+
You pass it a bare-metal connection to the database like `Mysql2::Client` (from `mysql2` gem on MRI) or `Java::OrgPostgresqlJdbc4::Jdbc4Connection` (from `jdbc-postgres` on Jruby).
|
8
|
+
|
9
|
+
As databases start to natively support SQL MERGE (which is basically upsert), this library will take advantage (but you won't have to change your code).
|
10
|
+
|
11
|
+
Does **not** depend on ActiveRecord.
|
12
|
+
|
13
|
+
Does **not** use `INSERT ON DUPLICATE KEY UPDATE` on MySQL as this only works if you are very careful about creating unique indexes.
|
14
|
+
|
15
|
+
70–90%+ faster than emulating upsert with ActiveRecord.
|
16
|
+
|
17
|
+
Supports MRI and JRuby.
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
You pass a __selector__ that uniquely identifies a row, whether it exists or not. You also pass a __setter__, attributes that should be set on that row.
|
22
|
+
|
23
|
+
Syntax inspired by [mongo-ruby-driver's update method](http://api.mongodb.org/ruby/1.6.4/Mongo/Collection.html#update-instance_method).
|
24
|
+
|
25
|
+
### Basic
|
26
|
+
|
27
|
+
```ruby
|
28
|
+
connection = Mysql2::Client.new([...])
|
29
|
+
table_name = :pets
|
30
|
+
upsert = Upsert.new connection, table_name
|
31
|
+
# N times...
|
32
|
+
upsert.row({:name => 'Jerry'}, :breed => 'beagle', :created_at => Time.now)
|
33
|
+
```
|
34
|
+
|
35
|
+
The `created_at` and `created_on` columns are used for inserts, but ignored on updates.
|
36
|
+
|
37
|
+
So just to reiterate you've got a __selector__ and a __setter__:
|
38
|
+
|
39
|
+
```ruby
|
40
|
+
selector = { :name => 'Jerry' }
|
41
|
+
setter = { :breed => 'beagle' }
|
42
|
+
upsert.row(selector, setter)
|
43
|
+
```
|
44
|
+
|
45
|
+
### Batch mode
|
46
|
+
|
47
|
+
By organizing your upserts into a batch, we can do work behind the scenes to make them faster.
|
48
|
+
|
49
|
+
```ruby
|
50
|
+
connection = Mysql2::Client.new([...])
|
51
|
+
Upsert.batch(connection, :pets) do |upsert|
|
52
|
+
# N times...
|
53
|
+
upsert.row({:name => 'Jerry'}, :breed => 'beagle')
|
54
|
+
upsert.row({:name => 'Pierre'}, :breed => 'tabby')
|
55
|
+
end
|
56
|
+
```
|
57
|
+
|
58
|
+
Batch mode is tested to be about 80% faster on PostgreSQL, MySQL, and SQLite3 than other ways to emulate upsert (see the tests, which fail if they are not faster).
|
59
|
+
|
60
|
+
### Native Postgres upsert
|
61
|
+
|
62
|
+
`INSERT ... ON CONFLICT DO UPDATE` is used when Postgres 9.5+ is detected and *unique constraint are in place.*
|
63
|
+
|
64
|
+
**Note: ** You must have a **unique constraint** on the column(s) you're using as a selector. A unique index won't work. See https://github.com/seamusabshere/upsert/issues/98#issuecomment-295341405 for more information and some ways to check.
|
65
|
+
|
66
|
+
If you don't have unique constraints, it will fall back to the classic Upsert gem user-defined function, which does not require a constraint.
|
67
|
+
|
68
|
+
### ActiveRecord helper method
|
69
|
+
|
70
|
+
```ruby
|
71
|
+
require 'upsert/active_record_upsert'
|
72
|
+
# N times...
|
73
|
+
Pet.upsert({:name => 'Jerry'}, :breed => 'beagle')
|
74
|
+
```
|
75
|
+
|
76
|
+
## Wishlist
|
77
|
+
|
78
|
+
Pull requests for any of these would be greatly appreciated:
|
79
|
+
|
80
|
+
1. Cache JDBC PreparedStatement objects.
|
81
|
+
1. Sanity check my three benchmarks (four if you include activerecord-import on MySQL). Do they accurately represent optimized alternatives?
|
82
|
+
1. Provide `require 'upsert/debug'` that will make sure you are selecting on columns that have unique indexes
|
83
|
+
1. Test that `Upsert` instances accept arbitrary columns, even within a batch, which is what people probably expect.
|
84
|
+
1. [@antage](https://github.com/antage)'s idea for "true" upserting: (from https://github.com/seamusabshere/upsert/issues/17)
|
85
|
+
|
86
|
+
```ruby
|
87
|
+
selector = { id: 15 }
|
88
|
+
update_setter = { count: Upsert.sql('count + 1') }
|
89
|
+
insert_setter = { count: 1 }
|
90
|
+
upsert.row_with_two_setter(update_setter, insert_setter, selector)
|
91
|
+
```
|
92
|
+
|
93
|
+
## Real-world usage
|
94
|
+
|
95
|
+
<p><a href="http://angel.co/faraday"><img src="https://s3.amazonaws.com/photos.angel.co/startups/i/175701-a63ebd1b56a401e905963c64958204d4-medium_jpg.jpg" alt="Faraday logo"/></a></p>
|
96
|
+
|
97
|
+
We use `upsert` for [big data at Faraday](http://angel.co/faraday). Originally written to speed up the [`data_miner`](https://github.com/seamusabshere/data_miner) data mining library.
|
98
|
+
|
99
|
+
## Supported databases/drivers
|
100
|
+
|
101
|
+
<table>
|
102
|
+
<tr>
|
103
|
+
<th>*</th>
|
104
|
+
<th>MySQL</th>
|
105
|
+
<th>PostgreSQL</th>
|
106
|
+
<th>SQLite3</th>
|
107
|
+
</tr>
|
108
|
+
<tr>
|
109
|
+
<th>MRI</th>
|
110
|
+
<td><a href="https://rubygems.org/gems/mysql2">mysql2</a></td>
|
111
|
+
<td><a href="https://rubygems.org/gems/pg">pg</a></td>
|
112
|
+
<td><a href="https://rubygems.org/gems/sqlite3">sqlite3</a></td>
|
113
|
+
</tr>
|
114
|
+
<tr>
|
115
|
+
<th>JRuby</th>
|
116
|
+
<td><a href="https://rubygems.org/gems/jdbc-mysql">jdbc-mysql</a></td>
|
117
|
+
<td><a href="https://rubygems.org/gems/jdbc-postgres">jdbc-postgres</a></td>
|
118
|
+
<td><a href="https://rubygems.org/gems/jdbc-sqlite3">jdbc-sqlite3</a></td>
|
119
|
+
</tr>
|
120
|
+
</table>
|
121
|
+
|
122
|
+
See below for details about what SQL MERGE trick (emulation of upsert) is used, performance, code examples, etc.
|
123
|
+
|
124
|
+
### Rails / ActiveRecord
|
125
|
+
|
126
|
+
(Assuming that one of the other three supported drivers is being used under the covers).
|
127
|
+
|
128
|
+
* add "upsert" to your Gemfile and
|
129
|
+
* run bundle install
|
130
|
+
|
131
|
+
```ruby
|
132
|
+
Upsert.new Pet.connection, Pet.table_name
|
133
|
+
```
|
134
|
+
|
135
|
+
#### Speed
|
136
|
+
|
137
|
+
Depends on the driver being used!
|
138
|
+
|
139
|
+
#### SQL MERGE trick
|
140
|
+
|
141
|
+
Depends on the driver being used!
|
142
|
+
|
143
|
+
### MySQL
|
144
|
+
|
145
|
+
On MRI, use the [mysql2](https://rubygems.org/gems/mysql2) driver.
|
146
|
+
|
147
|
+
```ruby
|
148
|
+
require 'mysql2'
|
149
|
+
connection = Mysql2::Connection.new(:username => 'root', :password => 'password', :database => 'upsert_test')
|
150
|
+
table_name = :pets
|
151
|
+
upsert = Upsert.new(connection, table_name)
|
152
|
+
```
|
153
|
+
|
154
|
+
On JRuby, use the [jdbc-mysql](https://rubygems.org/gems/jdbc-mysql) driver.
|
155
|
+
|
156
|
+
```ruby
|
157
|
+
require 'jdbc/mysql'
|
158
|
+
java.sql.DriverManager.register_driver com.mysql.jdbc.Driver.new
|
159
|
+
connection = java.sql.DriverManager.get_connection "jdbc:mysql://127.0.0.1/mydatabase?user=root&password=password"
|
160
|
+
```
|
161
|
+
|
162
|
+
#### Speed
|
163
|
+
|
164
|
+
From the tests (updated 11/7/12):
|
165
|
+
|
166
|
+
Upsert was 82% faster than find + new/set/save
|
167
|
+
Upsert was 85% faster than find_or_create + update_attributes
|
168
|
+
Upsert was 90% faster than create + rescue/find/update
|
169
|
+
Upsert was 46% faster than faking upserts with activerecord-import (note: in question as of 3/13/15, need some expert advice)
|
170
|
+
|
171
|
+
#### SQL MERGE trick
|
172
|
+
|
173
|
+
Thanks to [Dennis Hennen's StackOverflow response!](http://stackoverflow.com/questions/11371479/how-to-translate-postgresql-merge-db-aka-upsert-function-into-mysql/)!
|
174
|
+
|
175
|
+
```sql
|
176
|
+
CREATE PROCEDURE upsert_pets_SEL_name_A_tag_number_SET_name_A_tag_number(`name_sel` varchar(255), `tag_number_sel` int(11), `name_set` varchar(255), `tag_number_set` int(11))
|
177
|
+
BEGIN
|
178
|
+
DECLARE done BOOLEAN;
|
179
|
+
REPEAT
|
180
|
+
BEGIN
|
181
|
+
-- If there is a unique key constraint error then
|
182
|
+
-- someone made a concurrent insert. Reset the sentinel
|
183
|
+
-- and try again.
|
184
|
+
DECLARE ER_DUP_UNIQUE CONDITION FOR 23000;
|
185
|
+
DECLARE ER_INTEG CONDITION FOR 1062;
|
186
|
+
DECLARE CONTINUE HANDLER FOR ER_DUP_UNIQUE BEGIN
|
187
|
+
SET done = FALSE;
|
188
|
+
END;
|
189
|
+
|
190
|
+
DECLARE CONTINUE HANDLER FOR ER_INTEG BEGIN
|
191
|
+
SET done = TRUE;
|
192
|
+
END;
|
193
|
+
|
194
|
+
SET done = TRUE;
|
195
|
+
SELECT COUNT(*) INTO @count FROM `pets` WHERE `name` = `name_sel` AND `tag_number` = `tag_number_sel`;
|
196
|
+
-- Race condition here. If a concurrent INSERT is made after
|
197
|
+
-- the SELECT but before the INSERT below we'll get a duplicate
|
198
|
+
-- key error. But the handler above will take care of that.
|
199
|
+
IF @count > 0 THEN
|
200
|
+
-- UPDATE table_name SET b = b_SET WHERE a = a_SEL;
|
201
|
+
UPDATE `pets` SET `name` = `name_set`, `tag_number` = `tag_number_set` WHERE `name` = `name_sel` AND `tag_number` = `tag_number_sel`;
|
202
|
+
ELSE
|
203
|
+
-- INSERT INTO table_name (a, b) VALUES (k, data);
|
204
|
+
INSERT INTO `pets` (`name`, `tag_number`) VALUES (`name_set`, `tag_number_set`);
|
205
|
+
END IF;
|
206
|
+
END;
|
207
|
+
UNTIL done END REPEAT;
|
208
|
+
END
|
209
|
+
```
|
210
|
+
|
211
|
+
### PostgreSQL
|
212
|
+
|
213
|
+
On MRI, use the [pg](https://rubygems.org/gems/pg) driver.
|
214
|
+
|
215
|
+
```ruby
|
216
|
+
require 'pg'
|
217
|
+
connection = PG.connect(:dbname => 'upsert_test')
|
218
|
+
table_name = :pets
|
219
|
+
upsert = Upsert.new(connection, table_name)
|
220
|
+
```
|
221
|
+
|
222
|
+
On JRuby, use the [jdbc-postgres](https://rubygems.org/gems/jdbc-postgres) driver.
|
223
|
+
|
224
|
+
```ruby
|
225
|
+
require 'jdbc/postgres'
|
226
|
+
java.sql.DriverManager.register_driver org.postgresql.Driver.new
|
227
|
+
connection = java.sql.DriverManager.get_connection "jdbc:postgresql://127.0.0.1/mydatabase?user=root&password=password"
|
228
|
+
```
|
229
|
+
|
230
|
+
If you want to use HStore, make the `pg-hstore` gem available and pass a Hash in setters:
|
231
|
+
|
232
|
+
```ruby
|
233
|
+
gem 'pg-hstore'
|
234
|
+
require 'pg_hstore'
|
235
|
+
upsert.row({:name => 'Bill'}, :mydata => {:a => 1, :b => 2})
|
236
|
+
```
|
237
|
+
|
238
|
+
#### PostgreSQL notes
|
239
|
+
|
240
|
+
- Upsert doesn't do any type casting, so if you attempt to do something like the following:
|
241
|
+
`upsert.row({ :name => 'A Name' }, :tag_number => 'bob')`
|
242
|
+
you'll get an error which reads something like:
|
243
|
+
`invalid input syntax for integer: "bob"`
|
244
|
+
|
245
|
+
|
246
|
+
|
247
|
+
#### Speed
|
248
|
+
|
249
|
+
From the tests (updated 9/21/12):
|
250
|
+
|
251
|
+
Upsert was 72% faster than find + new/set/save
|
252
|
+
Upsert was 79% faster than find_or_create + update_attributes
|
253
|
+
Upsert was 83% faster than create + rescue/find/update
|
254
|
+
# (can't compare to activerecord-import because you can't fake it on pg)
|
255
|
+
|
256
|
+
#### SQL MERGE trick
|
257
|
+
|
258
|
+
Adapted from the [canonical PostgreSQL upsert example](http://www.postgresql.org/docs/current/interactive/plpgsql-control-structures.html#PLPGSQL-ERROR-TRAPPING):
|
259
|
+
|
260
|
+
```sql
|
261
|
+
CREATE OR REPLACE FUNCTION upsert_pets_SEL_name_A_tag_number_SET_name_A_tag_number("name_sel" character varying(255), "tag_number_sel" integer, "name_set" character varying(255), "tag_number_set" integer) RETURNS VOID AS
|
262
|
+
$$
|
263
|
+
DECLARE
|
264
|
+
first_try INTEGER := 1;
|
265
|
+
BEGIN
|
266
|
+
LOOP
|
267
|
+
-- first try to update the key
|
268
|
+
UPDATE "pets" SET "name" = "name_set", "tag_number" = "tag_number_set"
|
269
|
+
WHERE "name" = "name_sel" AND "tag_number" = "tag_number_sel";
|
270
|
+
IF found THEN
|
271
|
+
RETURN;
|
272
|
+
END IF;
|
273
|
+
-- not there, so try to insert the key
|
274
|
+
-- if someone else inserts the same key concurrently,
|
275
|
+
-- we could get a unique-key failure
|
276
|
+
BEGIN
|
277
|
+
INSERT INTO "pets"("name", "tag_number") VALUES ("name_set", "tag_number_set");
|
278
|
+
RETURN;
|
279
|
+
EXCEPTION WHEN unique_violation THEN
|
280
|
+
-- seamusabshere 9/20/12 only retry once
|
281
|
+
IF (first_try = 1) THEN
|
282
|
+
first_try := 0;
|
283
|
+
ELSE
|
284
|
+
RETURN;
|
285
|
+
END IF;
|
286
|
+
-- Do nothing, and loop to try the UPDATE again.
|
287
|
+
END;
|
288
|
+
END LOOP;
|
289
|
+
END;
|
290
|
+
$$
|
291
|
+
LANGUAGE plpgsql;
|
292
|
+
```
|
293
|
+
|
294
|
+
I slightly modified it so that it only retries once - don't want infinite loops.
|
295
|
+
|
296
|
+
### Sqlite
|
297
|
+
|
298
|
+
On MRI, use the [sqlite3](https://rubygems.org/gems/sqlite3) driver.
|
299
|
+
|
300
|
+
```ruby
|
301
|
+
require 'sqlite3'
|
302
|
+
connection = SQLite3::Database.open(':memory:')
|
303
|
+
table_name = :pets
|
304
|
+
upsert = Upsert.new(connection, table_name)
|
305
|
+
```
|
306
|
+
|
307
|
+
On JRuby, use the [jdbc-sqlite3](https://rubygems.org/gems/jdbc-sqlite3) driver.
|
308
|
+
|
309
|
+
```ruby
|
310
|
+
# TODO somebody please verify
|
311
|
+
require 'jdbc/sqlite3'
|
312
|
+
java.sql.DriverManager.register_driver org.sqlite.Driver.new
|
313
|
+
connection = java.sql.DriverManager.get_connection "jdbc:sqlite://127.0.0.1/mydatabase?user=root&password=password"
|
314
|
+
```
|
315
|
+
|
316
|
+
#### Speed
|
317
|
+
|
318
|
+
From the tests (updated 9/21/12):
|
319
|
+
|
320
|
+
Upsert was 77% faster than find + new/set/save
|
321
|
+
Upsert was 80% faster than find_or_create + update_attributes
|
322
|
+
Upsert was 85% faster than create + rescue/find/update
|
323
|
+
# (can't compare to activerecord-import because you can't fake it on sqlite3)
|
324
|
+
|
325
|
+
#### SQL MERGE trick
|
326
|
+
|
327
|
+
Thanks to [@dan04's answer on StackOverflow](http://stackoverflow.com/questions/2717590/sqlite-upsert-on-duplicate-key-update):
|
328
|
+
|
329
|
+
**Please note! This will only work properly on Sqlite if one of the columns being used as the "selector" are a primary key or unique index**
|
330
|
+
|
331
|
+
```sql
|
332
|
+
INSERT OR IGNORE INTO visits VALUES (127.0.0.1, 1);
|
333
|
+
UPDATE visits SET visits = 1 WHERE ip LIKE 127.0.0.1;
|
334
|
+
```
|
335
|
+
|
336
|
+
## Features
|
337
|
+
|
338
|
+
### Tested to be fast and portable
|
339
|
+
|
340
|
+
In addition to correctness, the library's tests check that it is
|
341
|
+
|
342
|
+
1. Faster than comparable upsert techniques
|
343
|
+
2. Compatible with supported databases
|
344
|
+
|
345
|
+
### Not dependent on ActiveRecord
|
346
|
+
|
347
|
+
As below, all you need is a raw database connection like a `Mysql2::Connection`, `PG::Connection` or a `SQLite3::Database`. These are equivalent:
|
348
|
+
|
349
|
+
```ruby
|
350
|
+
# with activerecord
|
351
|
+
Upsert.new ActiveRecord::Base.connection, :pets
|
352
|
+
# with activerecord, prettier
|
353
|
+
Upsert.new Pet.connection, Pet.table_name
|
354
|
+
# without activerecord
|
355
|
+
Upsert.new Mysql2::Connection.new([...]), :pets
|
356
|
+
```
|
357
|
+
|
358
|
+
### For a specific use case, faster and more portable than `activerecord-import`
|
359
|
+
|
360
|
+
You could also use [activerecord-import](https://github.com/zdennis/activerecord-import) to upsert:
|
361
|
+
|
362
|
+
```ruby
|
363
|
+
Pet.import columns, all_values, :timestamps => false, :on_duplicate_key_update => columns
|
364
|
+
```
|
365
|
+
|
366
|
+
`activerecord-import`, however, only works on MySQL and requires ActiveRecord—and if all you are doing is upserts, `upsert` is tested to be 40% faster. And you don't have to put all of the rows to be upserted into a single huge array - you can batch them using `Upsert.batch`.
|
367
|
+
|
368
|
+
## Gotchas
|
369
|
+
|
370
|
+
### No automatic typecasting beyond what the adapter/driver provides
|
371
|
+
|
372
|
+
We don't have any logic to convert integers into strings, strings into integers, etc. in order to satisfy PostgreSQL/etc.'s strictness on this issue.
|
373
|
+
|
374
|
+
So if you try to upsert a blank string (`''`) into an integer field in PostgreSQL, you will get an error.
|
375
|
+
|
376
|
+
### Dates and times are converted to UTC
|
377
|
+
|
378
|
+
Datetimes are immediately converted to UTC and sent to the database as ISO8601 strings.
|
379
|
+
|
380
|
+
If you're using MySQL, make sure server/connection timezone is UTC. If you're using Rails and/or ActiveRecord, you might want to check `ActiveRecord::Base.default_timezone`... it should probably be `:utc`.
|
381
|
+
|
382
|
+
In general, run some upserts and make sure datetimes get persisted like you expect.
|
383
|
+
|
384
|
+
### Clearning all library-generated functions
|
385
|
+
|
386
|
+
Place the following in to a rake task (so you don't globally redefine the `NAME_PREFIX` constant)
|
387
|
+
|
388
|
+
```ruby
|
389
|
+
Upsert::MergeFunction::NAME_PREFIX = "upsert"
|
390
|
+
|
391
|
+
# ActiveRecord
|
392
|
+
Upsert.clear_database_functions(ActiveRecord::Base.connection)
|
393
|
+
|
394
|
+
# Sequel
|
395
|
+
DB.synchronize do |conn|
|
396
|
+
Upsert.clear_database_functions(conn)
|
397
|
+
end
|
398
|
+
```
|
399
|
+
|
400
|
+
### Doesn't work with transactional fixtures
|
401
|
+
|
402
|
+
Per https://github.com/seamusabshere/upsert/issues/23 you might have issues if you try to use transactional fixtures and this library.
|
403
|
+
|
404
|
+
##
|
405
|
+
Testmetrics - https://www.testmetrics.app/seamusabshere/upsert
|
406
|
+
|
407
|
+
## Copyright
|
408
|
+
|
409
|
+
Copyright 2013-2019 Seamus Abshere
|
410
|
+
Copyright 2017-2019 Philip Schalm
|
411
|
+
Portions Copyright (c) 2019 The JRuby Team
|