sequel 5.43.0 → 5.47.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG +40 -0
  3. data/README.rdoc +1 -2
  4. data/doc/association_basics.rdoc +70 -11
  5. data/doc/migration.rdoc +11 -5
  6. data/doc/release_notes/5.44.0.txt +32 -0
  7. data/doc/release_notes/5.45.0.txt +34 -0
  8. data/doc/release_notes/5.46.0.txt +87 -0
  9. data/doc/release_notes/5.47.0.txt +59 -0
  10. data/doc/sql.rdoc +12 -0
  11. data/doc/testing.rdoc +5 -0
  12. data/doc/virtual_rows.rdoc +1 -1
  13. data/lib/sequel/adapters/odbc.rb +5 -1
  14. data/lib/sequel/adapters/shared/mysql.rb +17 -0
  15. data/lib/sequel/adapters/shared/postgres.rb +0 -12
  16. data/lib/sequel/adapters/shared/sqlite.rb +55 -9
  17. data/lib/sequel/core.rb +11 -0
  18. data/lib/sequel/database/schema_generator.rb +25 -46
  19. data/lib/sequel/database/schema_methods.rb +1 -1
  20. data/lib/sequel/dataset/query.rb +2 -4
  21. data/lib/sequel/dataset/sql.rb +7 -0
  22. data/lib/sequel/extensions/date_arithmetic.rb +29 -15
  23. data/lib/sequel/extensions/pg_enum.rb +1 -1
  24. data/lib/sequel/extensions/pg_loose_count.rb +3 -1
  25. data/lib/sequel/extensions/schema_dumper.rb +11 -0
  26. data/lib/sequel/model/associations.rb +275 -89
  27. data/lib/sequel/plugins/async_thread_pool.rb +1 -1
  28. data/lib/sequel/plugins/auto_validations_constraint_validations_presence_message.rb +68 -0
  29. data/lib/sequel/plugins/column_encryption.rb +20 -3
  30. data/lib/sequel/plugins/concurrent_eager_loading.rb +174 -0
  31. data/lib/sequel/plugins/many_through_many.rb +108 -9
  32. data/lib/sequel/plugins/pg_array_associations.rb +52 -38
  33. data/lib/sequel/plugins/prepared_statements.rb +10 -1
  34. data/lib/sequel/plugins/rcte_tree.rb +27 -19
  35. data/lib/sequel/plugins/timestamps.rb +1 -1
  36. data/lib/sequel/plugins/unused_associations.rb +520 -0
  37. data/lib/sequel/version.rb +1 -1
  38. metadata +14 -3
@@ -5,7 +5,7 @@ module Sequel
5
5
 
6
6
  module Plugins
7
7
  # The async_thread_pool plugin makes it slightly easier to use the async_thread_pool
8
- # Dataset extension with models. It makes Model.async return an async dataset for the
8
+ # Database extension with models. It makes Model.async return an async dataset for the
9
9
  # model, and support async behavior for #destroy, #with_pk, and #with_pk! for model
10
10
  # datasets:
11
11
  #
@@ -0,0 +1,68 @@
1
+ # frozen-string-literal: true
2
+
3
+ module Sequel
4
+ module Plugins
5
+ # The auto_validations_constraint_validations_presence_message plugin provides
6
+ # integration for the auto_validations and constraint_validations plugins in
7
+ # the following situation:
8
+ #
9
+ # * A column has a NOT NULL constraint in the database
10
+ # * A constraint validation for presence exists on the column, with a :message
11
+ # option to set a column-specific message, and with the :allow_nil option set
12
+ # to true because the CHECK constraint doesn't need to check for NULL values
13
+ # as the column itself is NOT NULL
14
+ #
15
+ # In this case, by default the validation error message on the column will
16
+ # use the more specific constraint validation error message if the column
17
+ # has a non-NULL empty value, but will use the default auto_validations
18
+ # message if the column has a NULL value. With this plugin, the column-specific
19
+ # constraint validation error message will be used in both cases.
20
+ #
21
+ # Usage:
22
+ #
23
+ # # Make all model subclasses use this auto_validations/constraint_validations
24
+ # # integration (called before loading subclasses)
25
+ # Sequel::Model.plugin :auto_validations_constraint_validations_presence_message
26
+ #
27
+ # # Make the Album class use this auto_validations/constraint_validations integration
28
+ # Album.plugin :auto_validations_constraint_validations_presence_message
29
+ module AutoValidationsConstraintValidationsPresenceMessage
30
+ def self.apply(model)
31
+ model.plugin :auto_validations
32
+ model.plugin :constraint_validations
33
+ end
34
+
35
+ def self.configure(model, opts=OPTS)
36
+ model.send(:_adjust_auto_validations_constraint_validations_presence_message)
37
+ end
38
+
39
+ module ClassMethods
40
+ Plugins.after_set_dataset(self, :_adjust_auto_validations_constraint_validations_presence_message)
41
+
42
+ private
43
+
44
+ def _adjust_auto_validations_constraint_validations_presence_message
45
+ if @dataset &&
46
+ !@auto_validate_options[:not_null][:message] &&
47
+ !@auto_validate_options[:explicit_not_null][:message]
48
+
49
+ @constraint_validations.each do |array|
50
+ meth, column, opts = array
51
+
52
+ if meth == :validates_presence &&
53
+ opts &&
54
+ opts[:message] &&
55
+ opts[:allow_nil] &&
56
+ (@auto_validate_not_null_columns.include?(column) || @auto_validate_explicit_not_null_columns.include?(column))
57
+
58
+ @auto_validate_not_null_columns.delete(column)
59
+ @auto_validate_explicit_not_null_columns.delete(column)
60
+ array[2] = array[2].merge(:allow_nil=>false)
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
@@ -7,10 +7,27 @@ raise(Sequel::Error, "Sequel column_encryption plugin requires ruby 2.3 or great
7
7
  require 'openssl'
8
8
 
9
9
  begin
10
- OpenSSL::Cipher.new("aes-256-gcm")
11
- rescue OpenSSL::Cipher::CipherError
10
+ # Test cipher actually works
11
+ cipher = OpenSSL::Cipher.new("aes-256-gcm")
12
+ cipher.encrypt
13
+ cipher.key = '1'*32
14
+ cipher_iv = cipher.random_iv
15
+ cipher.auth_data = ''
16
+ cipher_text = cipher.update('2') << cipher.final
17
+ auth_tag = cipher.auth_tag
18
+
19
+ cipher = OpenSSL::Cipher.new("aes-256-gcm")
20
+ cipher.decrypt
21
+ cipher.iv = cipher_iv
22
+ cipher.key = '1'*32
23
+ cipher.auth_data = ''
24
+ cipher.auth_tag = auth_tag
12
25
  # :nocov:
13
- raise LoadError, "Sequel column_encryption plugin requires the aes-256-gcm cipher"
26
+ unless (cipher.update(cipher_text) << cipher.final) == '2'
27
+ raise OpenSSL::Cipher::CipherError
28
+ end
29
+ rescue RuntimeError, OpenSSL::Cipher::CipherError
30
+ raise LoadError, "Sequel column_encryption plugin requires a working aes-256-gcm cipher"
14
31
  # :nocov:
15
32
  end
16
33
 
@@ -0,0 +1,174 @@
1
+ # frozen-string-literal: true
2
+
3
+ module Sequel
4
+ extension 'async_thread_pool'
5
+
6
+ module Plugins
7
+ # The concurrent_eager_loading plugin allows for eager loading multiple associations
8
+ # concurrently in separate threads. You must load the async_thread_pool Database
9
+ # extension into the Database object the model class uses in order for this plugin
10
+ # to work.
11
+ #
12
+ # By default in Sequel, eager loading happens in a serial manner. If you have code
13
+ # such as:
14
+ #
15
+ # Album.eager(:artist, :genre, :tracks)
16
+ #
17
+ # Sequel will load the albums, then the artists for the albums, then
18
+ # the genres for the albums, then the tracks for the albums.
19
+ #
20
+ # With the concurrent_eager_loading plugin, you can use the +eager_load_concurrently+
21
+ # method to allow for concurrent eager loading:
22
+ #
23
+ # Album.eager_load_concurrently.eager(:artist, :genre, :tracks)
24
+ #
25
+ # This will load the albums, first, since it needs to load the albums to know
26
+ # which artists, genres, and tracks to eagerly load. However, it will load the
27
+ # artists, genres, and tracks for the albums concurrently in separate threads.
28
+ # This can significantly improve performance, especially if there is significant
29
+ # latency between the application and the database. Note that using separate threads
30
+ # is only used in the case where there are multiple associations to eagerly load.
31
+ # With only a single association to eagerly load, there is no reason to use a
32
+ # separate thread, since it would not improve performance.
33
+ #
34
+ # If you want to make concurrent eager loading the default, you can load the
35
+ # plugin with the +:always+ option. In this case, all eager loads will be
36
+ # concurrent. If you want to force a non-concurrent eager load, you can use
37
+ # +eager_load_serially+:
38
+ #
39
+ # Album.eager_load_serially.eager(:artist, :genre, :tracks)
40
+ #
41
+ # Note that making concurrent eager loading the default is probably a bad idea
42
+ # if you are eager loading inside transactions and want the eager load to
43
+ # reflect changes made inside the transaction, unless you plan to use
44
+ # +eager_load_serially+ for such cases. See the async_thread_pool
45
+ # Database extension documentation for more general caveats regarding its use.
46
+ #
47
+ # The default eager loaders for all of the association types that ship with Sequel
48
+ # support safe concurrent eager loading. However, if you are specifying a custom
49
+ # +:eager_loader+ for an association, it may not work safely unless it it modified to
50
+ # support concurrent eager loading. Taking this example from the
51
+ # {Advanced Associations guide}[rdoc-ref:doc/advanced_associations.rdoc]
52
+ #
53
+ # Album.many_to_one :artist, :eager_loader=>(proc do |eo_opts|
54
+ # eo_opts[:rows].each{|album| album.associations[:artist] = nil}
55
+ # id_map = eo_opts[:id_map]
56
+ # Artist.where(:id=>id_map.keys).all do |artist|
57
+ # if albums = id_map[artist.id]
58
+ # albums.each do |album|
59
+ # album.associations[:artist] = artist
60
+ # end
61
+ # end
62
+ # end
63
+ # end)
64
+ #
65
+ # This would not support concurrent eager loading safely. To support safe
66
+ # concurrent eager loading, you need to make sure you are not modifying
67
+ # the associations for objects concurrently by separate threads. This is
68
+ # implemented using a mutex, which you can access via <tt>eo_opts[:mutex]</tt>.
69
+ # To keep things simple, you can use +Sequel.synchronize_with+ to only
70
+ # use this mutex if it is available. You want to use the mutex around the
71
+ # code that initializes the associations (usually to +nil+ or <tt>[]</tt>),
72
+ # and also around the code that sets the associatied objects appropriately
73
+ # after they have been retreived. You do not want to use the mutex around
74
+ # the code that loads the objects, since that will prevent concurrent loading.
75
+ # So after the changes, the custom eager loader would look like this:
76
+ #
77
+ # Album.many_to_one :artist, :eager_loader=>(proc do |eo_opts|
78
+ # Sequel.synchronize_with(eo[:mutex]) do
79
+ # eo_opts[:rows].each{|album| album.associations[:artist] = nil}
80
+ # end
81
+ # id_map = eo_opts[:id_map]
82
+ # rows = Artist.where(:id=>id_map.keys).all
83
+ # Sequel.synchronize_with(eo[:mutex]) do
84
+ # rows.each do |artist|
85
+ # if albums = id_map[artist.id]
86
+ # albums.each do |album|
87
+ # album.associations[:artist] = artist
88
+ # end
89
+ # end
90
+ # end
91
+ # end
92
+ # end)
93
+ #
94
+ # Usage:
95
+ #
96
+ # # Make all model subclass datasets support concurrent eager loading
97
+ # Sequel::Model.plugin :concurrent_eager_loading
98
+ #
99
+ # # Make the Album class datasets support concurrent eager loading
100
+ # Album.plugin :concurrent_eager_loading
101
+ #
102
+ # # Make all model subclass datasets concurrently eager load by default
103
+ # Sequel::Model.plugin :concurrent_eager_loading, always: true
104
+ module ConcurrentEagerLoading
105
+ def self.configure(mod, opts=OPTS)
106
+ if opts.has_key?(:always)
107
+ mod.instance_variable_set(:@always_eager_load_concurrently, opts[:always])
108
+ end
109
+ end
110
+
111
+ module ClassMethods
112
+ Plugins.inherited_instance_variables(self, :@always_eager_load_concurrently => nil)
113
+ Plugins.def_dataset_methods(self, [:eager_load_concurrently, :eager_load_serially])
114
+
115
+ # Whether datasets for this class should eager load concurrently by default.
116
+ def always_eager_load_concurrently?
117
+ @always_eager_load_concurrently
118
+ end
119
+ end
120
+
121
+ module DatasetMethods
122
+ # Return a cloned dataset that will eager load associated results concurrently
123
+ # using the async thread pool.
124
+ def eager_load_concurrently
125
+ cached_dataset(:_eager_load_concurrently) do
126
+ clone(:eager_load_concurrently=>true)
127
+ end
128
+ end
129
+
130
+ # Return a cloned dataset that will noteager load associated results concurrently
131
+ # using the async thread pool. Only useful if the current dataset has been marked
132
+ # as loading concurrently, or loading concurrently is the model's default behavior.
133
+ def eager_load_serially
134
+ cached_dataset(:_eager_load_serially) do
135
+ clone(:eager_load_concurrently=>false)
136
+ end
137
+ end
138
+
139
+ private
140
+
141
+ # Whether this particular dataset will eager load results concurrently.
142
+ def eager_load_concurrently?
143
+ v = @opts[:eager_load_concurrently]
144
+ v.nil? ? model.always_eager_load_concurrently? : v
145
+ end
146
+
147
+ # If performing eager loads concurrently, and at least 2 associations are being
148
+ # eagerly loaded, create a single mutex used for all eager loads. After the
149
+ # eager loads have been performed, force loading of any async results, so that
150
+ # all eager loads will have been completed before this method returns.
151
+ def perform_eager_loads(eager_load_data)
152
+ return super if !eager_load_concurrently? || eager_load_data.length < 2
153
+
154
+ mutex = Mutex.new
155
+ eager_load_data.each_value do |eo|
156
+ eo[:mutex] = mutex
157
+ end
158
+
159
+ super.each do |v|
160
+ if Sequel::Database::AsyncThreadPool::BaseProxy === v
161
+ v.__value
162
+ end
163
+ end
164
+ end
165
+
166
+ # If performing eager loads concurrently, perform this eager load using the
167
+ # async thread pool.
168
+ def perform_eager_load(loader, eo)
169
+ eo[:mutex] ? db.send(:async_run){super} : super
170
+ end
171
+ end
172
+ end
173
+ end
174
+ end
@@ -123,16 +123,25 @@ module Sequel
123
123
  nil
124
124
  end
125
125
 
126
+ # Whether a separate query should be used for each join table.
127
+ def separate_query_per_table?
128
+ self[:separate_query_per_table]
129
+ end
130
+
126
131
  private
127
132
 
128
133
  def _associated_dataset
129
134
  ds = associated_class
130
- (reverse_edges + [final_reverse_edge]).each do |t|
131
- h = {:qualify=>:deep}
132
- if t[:alias] != t[:table]
133
- h[:table_alias] = t[:alias]
135
+ if separate_query_per_table?
136
+ ds = ds.dataset
137
+ else
138
+ (reverse_edges + [final_reverse_edge]).each do |t|
139
+ h = {:qualify=>:deep}
140
+ if t[:alias] != t[:table]
141
+ h[:table_alias] = t[:alias]
142
+ end
143
+ ds = ds.join(t[:table], Array(t[:left]).zip(Array(t[:right])), h)
134
144
  end
135
- ds = ds.join(t[:table], Array(t[:left]).zip(Array(t[:right])), h)
136
145
  end
137
146
  ds
138
147
  end
@@ -208,6 +217,7 @@ module Sequel
208
217
  # :right (last array element) :: The key joining the table to the next table. Can use an
209
218
  # array of symbols for a composite key association.
210
219
  # If a hash is provided, the following keys are respected when using eager_graph:
220
+ # :db :: The Database containing the table. This changes lookup to use a separate query for each join table.
211
221
  # :block :: A proc to use as the block argument to join.
212
222
  # :conditions :: Extra conditions to add to the JOIN ON clause. Must be a hash or array of two pairs.
213
223
  # :join_type :: The join type to use for the join, defaults to :left_outer.
@@ -233,32 +243,121 @@ module Sequel
233
243
  opts[:after_load].unshift(:array_uniq!)
234
244
  end
235
245
  opts[:cartesian_product_number] ||= one_through_many ? 0 : 2
236
- opts[:through] = opts[:through].map do |e|
246
+ separate_query_per_table = false
247
+ through = opts[:through] = opts[:through].map do |e|
237
248
  case e
238
249
  when Array
239
250
  raise(Error, "array elements of the through option/argument for many_through_many associations must have at least three elements") unless e.length == 3
240
251
  {:table=>e[0], :left=>e[1], :right=>e[2]}
241
252
  when Hash
242
253
  raise(Error, "hash elements of the through option/argument for many_through_many associations must contain :table, :left, and :right keys") unless e[:table] && e[:left] && e[:right]
254
+ separate_query_per_table = true if e[:db]
243
255
  e
244
256
  else
245
257
  raise(Error, "the through option/argument for many_through_many associations must be an enumerable of arrays or hashes")
246
258
  end
247
259
  end
260
+ opts[:separate_query_per_table] = separate_query_per_table
248
261
 
249
262
  left_key = opts[:left_key] = opts[:through].first[:left]
250
263
  opts[:left_keys] = Array(left_key)
251
- opts[:uses_left_composite_keys] = left_key.is_a?(Array)
264
+ uses_lcks = opts[:uses_left_composite_keys] = left_key.is_a?(Array)
252
265
  left_pk = (opts[:left_primary_key] ||= self.primary_key)
253
266
  raise(Error, "no primary key specified for #{inspect}") unless left_pk
254
267
  opts[:eager_loader_key] = left_pk unless opts.has_key?(:eager_loader_key)
255
268
  opts[:left_primary_keys] = Array(left_pk)
256
269
  lpkc = opts[:left_primary_key_column] ||= left_pk
257
270
  lpkcs = opts[:left_primary_key_columns] ||= Array(lpkc)
258
- opts[:dataset] ||= opts.association_dataset_proc
259
271
 
260
272
  opts[:left_key_alias] ||= opts.default_associated_key_alias
261
- opts[:eager_loader] ||= opts.method(:default_eager_loader)
273
+ if separate_query_per_table
274
+ opts[:use_placeholder_loader] = false
275
+ opts[:allow_eager_graph] = false
276
+ opts[:allow_filtering_by] = false
277
+ opts[:eager_limit_strategy] = nil
278
+
279
+ opts[:dataset] ||= proc do |r|
280
+ def_db = r.associated_class.db
281
+ vals = uses_lcks ? [lpkcs.map{|k| get_column_value(k)}] : get_column_value(left_pk)
282
+
283
+ has_results = through.each do |edge|
284
+ ds = (edge[:db] || def_db).from(edge[:table]).where(edge[:left]=>vals)
285
+ ds = ds.where(edge[:conditions]) if edge[:conditions]
286
+ right = edge[:right]
287
+ vals = ds.select_map(right)
288
+ if right.is_a?(Array)
289
+ vals.delete_if{|v| v.any?(&:nil?)}
290
+ else
291
+ vals.delete(nil)
292
+ end
293
+ break if vals.empty?
294
+ end
295
+
296
+ ds = r.associated_dataset.where(opts.right_primary_key=>vals)
297
+ ds = ds.clone(:no_results=>true) unless has_results
298
+ ds
299
+ end
300
+ opts[:eager_loader] ||= proc do |eo|
301
+ h = eo[:id_map]
302
+ assign_singular = opts.assign_singular?
303
+ uses_rcks = opts.right_primary_key.is_a?(Array)
304
+ rpk = uses_rcks ? opts.right_primary_keys : opts.right_primary_key
305
+ name = opts[:name]
306
+ def_db = opts.associated_class.db
307
+ join_map = h
308
+
309
+ run_query = through.each do |edge|
310
+ ds = (edge[:db] || def_db).from(edge[:table])
311
+ ds = ds.where(edge[:conditions]) if edge[:conditions]
312
+ left = edge[:left]
313
+ right = edge[:right]
314
+ prev_map = join_map
315
+ join_map = ds.where(left=>join_map.keys).select_hash_groups(right, left)
316
+ if right.is_a?(Array)
317
+ join_map.delete_if{|v,| v.any?(&:nil?)}
318
+ else
319
+ join_map.delete(nil)
320
+ end
321
+ break if join_map.empty?
322
+ join_map.each_value do |vs|
323
+ vs.replace(vs.flat_map{|v| prev_map[v]})
324
+ vs.uniq!
325
+ end
326
+ end
327
+
328
+ eo = Hash[eo]
329
+
330
+ if run_query
331
+ eo[:loader] = false
332
+ eo[:right_keys] = join_map.keys
333
+ else
334
+ eo[:no_results] = true
335
+ end
336
+
337
+ opts[:model].eager_load_results(opts, eo) do |assoc_record|
338
+ rpkv = if uses_rcks
339
+ assoc_record.values.values_at(*rpk)
340
+ else
341
+ assoc_record.values[rpk]
342
+ end
343
+
344
+ objects = join_map[rpkv]
345
+
346
+ if assign_singular
347
+ objects.each do |object|
348
+ object.associations[name] ||= assoc_record
349
+ end
350
+ else
351
+ objects.each do |object|
352
+ object.associations[name].push(assoc_record)
353
+ end
354
+ end
355
+ end
356
+ end
357
+ else
358
+ opts[:dataset] ||= opts.association_dataset_proc
359
+ opts[:eager_loader] ||= opts.method(:default_eager_loader)
360
+ end
262
361
 
263
362
  join_type = opts[:graph_join_type]
264
363
  select = opts[:graph_select]