sequel 5.46.0 → 5.47.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8202d77fff48270013e8d06b75c5ab51933ff9e3fda72f99139211c9cb00de65
-  data.tar.gz: 15393a6189c83eb324e243d81805da38deced274d3f3a1b64bbf3cee54a27fa6
+  metadata.gz: 191a57b6bd41c00e023891afaddbebffeb1f54017c663d2a9b32faf83584bf1f
+  data.tar.gz: d158aa702dfe28dbe624b551ed455615017942caf990c599f077a4fb53f6b533
 SHA512:
-  metadata.gz: 35aa602f835100be3a01bec05ecfb3a0d53555774d1dff520963c254d79c8123328f61e2252c8cb9c69b935f015de7c53f5cbeaec378d425666e7c96bcd0dba7
-  data.tar.gz: b52d0a0e2ea2fe6e388bd8fc694bac86d66f960cba0ed6c952213ba2414395862d3ba734b1763f13883ec7f7983cf69fe75a39a0f8e9406af46e2a2974f7210e
+  metadata.gz: 5c91ce33e0191d342f34dbeb4a402153bedc6d46a1df56d99b1393cf559bb9edd837e3eca8f259e1f7856990015e40098cbb269cbc9e915fa4dac9fc254c8e0c
+  data.tar.gz: 9d789484e942ac7380e6dbae64b3e0fb27aeaf9d3fc6764f2e6d6e86c17296cdab116cd6c8d53c429044604a429fdaf18536b6f68b728f5031a81aba81f4107b

data/CHANGELOG

@@ -1,3 +1,17 @@
+=== 5.47.0 (2021-08-01)
+
+* Make the unused_associations plugin track access to association reflections to determine whether associations are used (jeremyevans)
+
+* Support :db option for join tables in {many,one}_through_many to use a separate query for each join table (jeremyevans)
+
+* Support :join_table_db option for many_to_many/one_through_one associations, to use a separate query for the join table (jeremyevans)
+
+* Support :allow_eager_graph and :allow_filtering_by association options (jeremyevans)
+
+* Add Database#rename_tables on MySQL, for renaming multiple tables in a single call (nick96) (#1774)
+
+* Support Dataset#returning on SQLite 3.35+ (jeremyevans)
+
 === 5.46.0 (2021-07-01)
 
 * Add unused_associations plugin, for determining which associations and association methods are not used (jeremyevans)

data/doc/association_basics.rdoc

@@ -41,7 +41,7 @@ As is the code to add a related album to an artist:
 
   @artist.add_album(name: 'RF')
 
-It also makes it easier to creating queries that use joins based on the association:
+It also makes it easier to create queries that use joins based on the association:
 
   Artist.association_join(:albums)
   # SELECT * FROM artists
@@ -63,8 +63,8 @@ It ships with additional association types via plugins.
 
 The many_to_one association is used when the table for the current class
 contains a foreign key that references the primary key in the table for the
-associated class.  It is named because there can be many rows in the current
-table for each row in the associated table.
+associated class.  It is named 'many_to_one' because there can be many rows
+in the current table for each row in the associated table.
 
   # Database schema:
   #  albums             artists
@@ -81,8 +81,8 @@ table for each row in the associated table.
 
 The one_to_many association is used when the table for the associated class
 contains a foreign key that references the primary key in the table for the
-current class.  It is named because for each row in the current table there
-can be many rows in the associated table:
+current class.  It is named 'one_to_many' because for each row in the
+current table there can be many rows in the associated table:
 
 The one_to_one association can be thought of as a subset of the one_to_many association,
 but where there can only be either 0 or 1 records in the associated table.  This is
@@ -344,7 +344,7 @@ instance method:
 
 == Dataset Method
 
-In addition to the above methods, associations also add a instance method
+In addition to the above methods, associations also add an instance method
 ending in +_dataset+ that returns a dataset representing the objects in the associated table:
 
   @album.artist_id
@@ -1107,7 +1107,7 @@ already applied, and the proc should return a modified copy of this dataset.
 Here's an example of an association of songs to artists through lyrics, where
 the artist can perform any one of four tasks for the lyric:
 
-  Album.one_to_many :songs, dataset: (lambda do |r|
+  Artist.one_to_many :songs, dataset: (lambda do |r|
     r.associated_dataset.select_all(:songs).
      join(:lyrics, id: :lyricid, id=>[:composer_id, :arranger_id, :vocalist_id, :lyricist_id])
   end)
@@ -1166,6 +1166,23 @@ when deleting.
     ds.where(instrument_id: 5)
   end)
 
+==== :join_table_db [+many_to_many+, +one_through_one+]
+
+A Sequel::Database to use for the join table.  Specifying this option switches the
+loading to use a separate query for the join table.  This is useful if the
+join table is not located in the same database as the associated table, or
+if the database account with access to the associated table doesn't have
+access to the join table.
+
+For example, if the Album class uses a different Sequel::Database than the Artist
+class, and the join table is in the database that the Artist class uses:
+
+  Artist.many_to_many :lead_guitar_albums, class: :Album, :join_table_db=>Artist.db
+
+This option also affects the add/remove/remove_all methods, by changing
+which database is used for inserts/deletes from the join table (add/remove/remove_all
+defaults to use the current model's database instead of the associated model's database).
+
 === Callback Options
 
 All callbacks can be specified as a Symbol, Proc, or array of both/either
@@ -1686,12 +1703,35 @@ If set to false, you cannot load the association eagerly via eager or
 eager_graph.
 
   Artist.one_to_many :albums, allow_eager: false
-  Artist.eager(:albums) # Raises Sequel::Error
+  Artist.eager(:albums)       # Raises Sequel::Error
+  Artist.eager_graph(:albums) # Raises Sequel::Error
 
 This is usually used if the association dataset depends on specific values in
 model instance that would not be valid when eager loading for multiple
 instances.
 
+==== :allow_eager_graph
+
+If set to false, you cannot load the association eagerly via eager_graph.
+
+  Artist.one_to_many :albums, allow_eager_graph: false
+  Artist.eager(:albums)       # Allowed
+  Artist.eager_graph(:albums) # Raises Sequel::Error
+
+This is useful if you still want to allow loading via eager, but do not want
+to allow loading via eager graph, possibly because the association does not
+support joins.
+
+==== :allow_filtering_by
+
+If set to false, you cannot use the association when filtering.
+
+  Artist.one_to_many :albums, allow_filtering_by: false
+  Artist.where(:albums=>Album.where(:name=>'A')).all # Raises Sequel::Error
+
+This is useful if such filtering cannot work, such as when a subquery cannot
+be used because the necessary tables are not in the same database.
+
 ==== :instance_specific
 
 This allows you to override the setting of whether the dataset contains instance

data/doc/migration.rdoc

@@ -543,16 +543,22 @@ The main difference between the two is that <tt>-d</tt> will use the type method
 with the database independent ruby class types, while <tt>-D</tt> will use
 the +column+ method with string types.
 
-Note that Sequel cannot dump constraints other than primary key and possibly
-foreign key constraints.  If you are using database features such
-as constraints or triggers, you should use your database's dump and restore
-programs instead of Sequel's schema dumper.
-
 You can take the migration created by the schema dumper to another computer
 with an empty database, and attempt to recreate the schema using:
 
   sequel -m db/migrations postgres://host/database
 
+The schema_dumper extension is quite limited in what types of
+database objects it supports.  In general, it only supports
+dumping tables, columns, primary key and foreign key constraints,
+and some indexes.  It does not support most table options, CHECK
+constraints, partial indexes, database functions, triggers,
+security grants/revokes, and a wide variety of other useful
+database properties.  Be aware of the limitations when using the
+schema_dumper extension. If you are dumping the schema to restore
+to the same database type, it is recommended to use your database's
+dump and restore programs instead of the schema_dumper extension.
+
 == Checking for Current Migrations
 
 In your application code, you may want to check that you are up to date in

data/doc/release_notes/5.47.0.txt

@@ -0,0 +1,59 @@
+= New Features
+
+* Sequel now supports using separate queries for each table for both
+  lazy and eager loading of the following associations:
+
+  * many_to_many
+  * one_through_one
+  * many_through_many # many_through_many plugin
+  * one_through_many # many_through_many plugin
+
+  For many_to_many/one_through_one, you specify the :join_table_db
+  association option, which should be a Sequel::Database instance
+  containing the join table.  It is possible for the current table,
+  join table, and associated table all to be in separate databases:
+
+    JOIN_TABLE_DB = Sequel.connect('...')
+    Album.many_to_many :artists, join_table_db: JOIN_TABLE_DB
+
+  For many_through_many/one_through_many, you can use the :db option
+  in each join table specification.  All join tables can be in
+  separate databases:
+
+    JTDB1 = Sequel.connect('...')
+    JTDB2 = Sequel.connect('...')
+    # Tracks on all albums this artist appears on
+    Artist.many_through_many :album_tracks, [
+        {table: :albums_artists, left: :artist_id, right: :album_id, db: JTDB1},
+        {table: :artists, left: :id, right: :id, db: JTDB2}
+      ],
+      class: :Track, right_primary_key: :album_id
+
+* The :allow_eager_graph association option has been added. Setting
+  this option to false will disallow eager loading via #eager_graph.
+  This is useful if you can eager load the association via #eager,
+  but not with #eager_graph.
+
+* The :allow_filtering_by association option has been added. Setting
+  this option to false will disallow the use of filtering by
+  associations for the association.
+
+* Dataset#returning is now supported on SQLite 3.35.0+. To work around
+  bugs in the SQLite implementation, identifiers used in the RETURNING
+  clause are automatically aliased.  Additionally, prepared statements
+  that use the RETURNING clause on SQLite seem to have issues, so the
+  prepared_statements plugin does not automatically use prepared
+  statements on SQLite for queries that use the RETURNING clause.
+
+* Database#rename_tables has been added on MySQL to support renaming
+  multiple tables in the same query.
+
+= Other Improvements
+
+* The unused_associations plugin now tracks access to the association
+  reflection for associations, so it will no longer show an
+  association as completely unused if something is accessing the
+  association reflection for it.  This eliminates most of the false
+  positives, where the plugin would show an association as unused
+  even though something was using it without calling the association
+  methods.

data/doc/sql.rdoc

@@ -428,6 +428,18 @@ As you can see, these literalize with ANDs by default.  You can use the <tt>Sequ
 
   Sequel.or(column1: 1, column2: 2) # (("column1" = 1) OR ("column2" = 2))
 
+As you can see in the above examples, <tt>Sequel.|</tt> and <tt>Sequel.or</tt> work differently.
+<tt>Sequel.|</tt> is for combining an arbitrary number of expressions using OR. If you pass a single
+argument, <tt>Sequel.|</tt> will just convert it to a Sequel expression, similar to <tt>Sequel.expr</tt>.
+<tt>Sequel.or</tt> is for taking a single hash or array of two element arrays and combining the
+elements of that single argument using OR instead of AND:
+
+  Sequel.|(column1: 1, column2: 2) # (("column1" = 1) AND ("column2" = 2))
+  Sequel.or(column1: 1, column2: 2) # (("column1" = 1) OR ("column2" = 2))
+
+  Sequel.|({column1: 1}, {column2: 2}) # (("column1" = 1) OR ("column2" = 2))
+  Sequel.or({column1: 1}, {column2: 2}) # ArgumentError
+
 You've already seen the <tt>Sequel.negate</tt> method, which will use ANDs if multiple entries are used:
 
   Sequel.negate(column1: 1, column2: 2) # (("column1" != 1) AND ("column2" != 2))

data/doc/testing.rdoc

@@ -175,6 +175,10 @@ SEQUEL_NO_CACHE_ASSOCIATIONS :: Don't cache association metadata when running th
 SEQUEL_CHECK_PENDING :: Try running all specs (note, can cause lockups for some adapters), and raise errors for skipped specs that don't fail
 SEQUEL_NO_PENDING :: Don't skip any specs, try running all specs (note, can cause lockups for some adapters)
 SEQUEL_PG_TIMESTAMPTZ :: Use the pg_timestamptz extension when running the postgres specs
+SEQUEL_QUERY_PER_ASSOCIATION_DB_0_URL :: Run query-per-association integration tests with multiple databases (all 4 must be set to run)
+SEQUEL_QUERY_PER_ASSOCIATION_DB_1_URL :: Run query-per-association integration tests with multiple databases (all 4 must be set to run)
+SEQUEL_QUERY_PER_ASSOCIATION_DB_2_URL :: Run query-per-association integration tests with multiple databases (all 4 must be set to run)
+SEQUEL_QUERY_PER_ASSOCIATION_DB_3_URL :: Run query-per-association integration tests with multiple databases (all 4 must be set to run)
 SEQUEL_SPLIT_SYMBOLS :: Turn on symbol splitting when running the adapter and integration specs
 SEQUEL_SYNCHRONIZE_SQL :: Use the synchronize_sql extension when running the specs
 SEQUEL_TZINFO_VERSION :: Force the given tzinfo version when running the specs (e.g. '>=2')

data/lib/sequel/adapters/shared/mysql.rb

@@ -187,6 +187,15 @@ module Sequel
       def views(opts=OPTS)
         full_tables('VIEW', opts)
       end
+
+      # Renames multiple tables in a single call.
+      #
+      #   DB.rename_tables [:items, :old_items], [:other_items, :old_other_items]
+      #   # RENAME TABLE items TO old_items, other_items TO old_other_items
+      def rename_tables(*renames)
+        execute_ddl(rename_tables_sql(renames))
+        renames.each{|from,| remove_cached_schema(from)}
+      end
       
       private
       
@@ -473,6 +482,14 @@ module Sequel
         schema(table).select{|a| a[1][:primary_key]}.map{|a| a[0]}
       end
 
+      # SQL statement for renaming multiple tables.
+      def rename_tables_sql(renames)
+        rename_tos = renames.map do |from, to|
+            "#{quote_schema_table(from)} TO #{quote_schema_table(to)}"
+        end.join(', ')
+        "RENAME TABLE #{rename_tos}"
+      end
+
       # Rollback the currently open XA transaction
       def rollback_transaction(conn, opts=OPTS)
         if (s = opts[:prepare]) && savepoint_level(conn) <= 1

data/lib/sequel/adapters/shared/sqlite.rb

@@ -562,10 +562,10 @@ module Sequel
       EXTRACT_MAP = {:year=>"'%Y'", :month=>"'%m'", :day=>"'%d'", :hour=>"'%H'", :minute=>"'%M'", :second=>"'%f'"}.freeze
       EXTRACT_MAP.each_value(&:freeze)
 
-      Dataset.def_sql_method(self, :delete, [['if db.sqlite_version >= 30803', %w'with delete from where'], ["else", %w'delete from where']])
-      Dataset.def_sql_method(self, :insert, [['if db.sqlite_version >= 30803', %w'with insert conflict into columns values on_conflict'], ["else", %w'insert conflict into columns values']])
+      Dataset.def_sql_method(self, :delete, [['if db.sqlite_version >= 33500', %w'with delete from where returning'], ['elsif db.sqlite_version >= 30803', %w'with delete from where'], ["else", %w'delete from where']])
+      Dataset.def_sql_method(self, :insert, [['if db.sqlite_version >= 33500', %w'with insert conflict into columns values on_conflict returning'], ['elsif db.sqlite_version >= 30803', %w'with insert conflict into columns values on_conflict'], ["else", %w'insert conflict into columns values']])
       Dataset.def_sql_method(self, :select, [['if opts[:values]', %w'with values compounds'], ['else', %w'with select distinct columns from join where group having window compounds order limit lock']])
-      Dataset.def_sql_method(self, :update, [['if db.sqlite_version >= 33300', %w'with update table set from where'], ['elsif db.sqlite_version >= 30803', %w'with update table set where'], ["else", %w'update table set where']])
+      Dataset.def_sql_method(self, :update, [['if db.sqlite_version >= 33500', %w'with update table set from where returning'], ['elsif db.sqlite_version >= 33300', %w'with update table set from where'], ['elsif db.sqlite_version >= 30803', %w'with update table set where'], ["else", %w'update table set where']])
 
       def cast_sql_append(sql, expr, type)
         if type == Time or type == DateTime
@@ -639,8 +639,8 @@ module Sequel
       # SQLite performs a TRUNCATE style DELETE if no filter is specified.
       # Since we want to always return the count of records, add a condition
       # that is always true and then delete.
-      def delete
-        @opts[:where] ? super : where(1=>1).delete
+      def delete(&block)
+        @opts[:where] ? super : where(1=>1).delete(&block)
       end
       
       # Return an array of strings specifying a query explanation for a SELECT of the
@@ -661,6 +661,21 @@ module Sequel
         super
       end
       
+      # Support insert select for associations, so that the model code can use
+      # returning instead of a separate query.
+      def insert_select(*values)
+        return unless supports_insert_select?
+        # Handle case where query does not return a row
+        server?(:default).with_sql_first(insert_select_sql(*values)) || false
+      end
+
+      # The SQL to use for an insert_select, adds a RETURNING clause to the insert
+      # unless the RETURNING clause is already present.
+      def insert_select_sql(*values)
+        ds = opts[:returning] ? self : returning
+        ds.insert_sql(*values)
+      end
+
       # SQLite uses the nonstandard ` (backtick) for quoting identifiers.
       def quoted_identifier_append(sql, c)
         sql << '`' << c.to_s.gsub('`', '``') << '`'
@@ -742,6 +757,13 @@ module Sequel
         insert_conflict(:ignore)
       end
 
+      # Automatically add aliases to RETURNING values to work around SQLite bug.
+      def returning(*values)
+        return super if values.empty?
+        raise Error, "RETURNING is not supported on #{db.database_type}" unless supports_returning?(:insert)
+        clone(:returning=>_returning_values(values).freeze)
+      end
+
       # SQLite 3.8.3+ supports common table expressions.
       def supports_cte?(type=:select)
         db.sqlite_version >= 30803
@@ -782,6 +804,11 @@ module Sequel
         false
       end
       
+      # SQLite 3.35.0 supports RETURNING on INSERT/UPDATE/DELETE.
+      def supports_returning?(_)
+        db.sqlite_version >= 33500
+      end
+
       # SQLite supports timezones in literal timestamps, since it stores them
       # as text.  But using timezones in timestamps breaks SQLite datetime
       # functions, so we allow the user to override the default per database.
@@ -814,6 +841,21 @@ module Sequel
 
       private
       
+      # Add aliases to symbols and identifiers to work around SQLite bug.
+      def _returning_values(values)
+        values.map do |v|
+          case v
+          when Symbol
+            _, c, a = split_symbol(v)
+            a ? v : Sequel.as(v, c)
+          when SQL::Identifier, SQL::QualifiedIdentifier
+            Sequel.as(v, unqualified_column_for(v))
+          else
+            v
+          end
+        end
+      end
+
       # SQLite uses string literals instead of identifiers in AS clauses.
       def as_sql_append(sql, aliaz, column_aliases=nil)
         raise Error, "sqlite does not support derived column lists" if column_aliases

data/lib/sequel/database/schema_methods.rb

@@ -63,7 +63,7 @@ module Sequel
     # definitions using <tt>create_table</tt>, and +add_index+ accepts all the options
     # available for index definition.
     #
-    # See <tt>Schema::AlterTableGenerator</tt> and the {"Migrations and Schema Modification" guide}[rdoc-ref:doc/migration.rdoc].
+    # See <tt>Schema::AlterTableGenerator</tt> and the {Migrations guide}[rdoc-ref:doc/migration.rdoc].
     def alter_table(name, &block)
       generator = alter_table_generator(&block)
       remove_cached_schema(name)

data/lib/sequel/dataset/query.rb

@@ -699,7 +699,7 @@ module Sequel
     end
 
     # Returns a copy of the dataset with a specified order. Can be safely combined with limit.
-    # If you call limit with an offset, it will override override the offset if you've called
+    # If you call limit with an offset, it will override the offset if you've called
     # offset first.
     #
     #   DB[:items].offset(10) # SELECT * FROM items OFFSET 10

data/lib/sequel/extensions/schema_dumper.rb

@@ -6,6 +6,17 @@
 # the current database).  The main interface is through
 # Sequel::Database#dump_schema_migration.
 #
+# The schema_dumper extension is quite limited in what types of
+# database objects it supports.  In general, it only supports
+# dumping tables, columns, primary key and foreign key constraints,
+# and some indexes.  It does not support most table options, CHECK
+# constraints, partial indexes, database functions, triggers,
+# security grants/revokes, and a wide variety of other useful
+# database properties.  Be aware of the limitations when using the
+# schema_dumper extension. If you are dumping the schema to restore
+# to the same database type, it is recommended to use your database's
+# dump and restore programs instead of the schema_dumper extension.
+#
 # To load the extension:
 #
 #   DB.extension :schema_dumper

data/lib/sequel/model/associations.rb

@@ -274,7 +274,9 @@ module Sequel
           cascade = eo[:associations]
           eager_limit = nil
 
-          if eo[:eager_block] || eo[:loader] == false
+          if eo[:no_results]
+            no_results = true
+          elsif eo[:eager_block] || eo[:loader] == false
             ds = eager_loading_dataset(eo)
 
             strategy = ds.opts[:eager_limit_strategy] || strategy
@@ -313,7 +315,7 @@ module Sequel
             objects = loader.all(ids)
           end
 
-          Sequel.synchronize_with(eo[:mutex]){objects.each(&block)}
+          Sequel.synchronize_with(eo[:mutex]){objects.each(&block)} unless no_results
 
           if strategy == :ruby
             apply_ruby_eager_limit_strategy(rows, eager_limit || limit_and_offset)
@@ -638,9 +640,7 @@ module Sequel
         # given the hash passed to the eager loader.
         def eager_loading_dataset(eo=OPTS)
           ds = eo[:dataset] || associated_eager_dataset
-          if id_map = eo[:id_map]
-            ds = ds.where(eager_loading_predicate_condition(id_map.keys))
-          end
+          ds = eager_loading_set_predicate_condition(ds, eo)
           if associations = eo[:associations]
             ds = ds.eager(associations)
           end
@@ -667,6 +667,15 @@ module Sequel
           self[:model].default_eager_limit_strategy || :ruby
         end
 
+        # Set the predicate condition for the eager loading dataset based on the id map
+        # in the eager loading options.
+        def eager_loading_set_predicate_condition(ds, eo)
+          if id_map = eo[:id_map]
+            ds = ds.where(eager_loading_predicate_condition(id_map.keys))
+          end
+          ds
+        end
+
         # The predicate condition to use for the eager_loader.
         def eager_loading_predicate_condition(keys)
           {predicate_key=>keys}
@@ -1318,7 +1327,7 @@ module Sequel
     
         # many_to_many associations need to select a key in an associated table to eagerly load
         def eager_loading_use_associated_key?
-          true
+          !separate_query_per_table?
         end
 
         # The source of the join table.  This is the join table itself, unless it
@@ -1375,10 +1384,30 @@ module Sequel
           cached_fetch(:select){default_select}
         end
 
+        # Whether a separate query should be used for the join table.
+        def separate_query_per_table?
+          self[:join_table_db]
+        end
+
         private
 
+        # Join to the the join table, unless using a separate query per table.
         def _associated_dataset
-          super.inner_join(self[:join_table], self[:right_keys].zip(right_primary_keys), :qualify=>:deep)
+          if separate_query_per_table?
+            super
+          else
+            super.inner_join(self[:join_table], self[:right_keys].zip(right_primary_keys), :qualify=>:deep)
+          end
+        end
+
+        # Use the right_keys from the eager loading options if
+        # using a separate query per table.
+        def eager_loading_set_predicate_condition(ds, eo)
+          if separate_query_per_table?
+            ds.where(right_primary_key=>eo[:right_keys])
+          else
+            super
+          end
         end
 
         # The default selection for associations that require joins.  These do not use the default
@@ -1606,6 +1635,8 @@ module Sequel
         #               after an item is set using the association setter method.
         # :allow_eager :: If set to false, you cannot load the association eagerly
         #                 via eager or eager_graph
+        # :allow_eager_graph :: If set to false, you cannot load the association eagerly via eager_graph.
+        # :allow_filtering_by :: If set to false, you cannot use the association when filtering
         # :before_add :: Symbol, Proc, or array of both/either specifying a callback to call
         #                before a new item is added to the association.
         # :before_remove :: Symbol, Proc, or array of both/either specifying a callback to call
@@ -1773,6 +1804,9 @@ module Sequel
         #                underscored, sorted, and joined with '_'.
         # :join_table_block :: proc that can be used to modify the dataset used in the add/remove/remove_all
         #                      methods.  Should accept a dataset argument and return a modified dataset if present.
+        # :join_table_db :: When retrieving records when using lazy loading or eager loading via +eager+, instead of
+        #                   a join between to the join table and the associated table, use a separate query for the
+        #                   join table using the given Database object.
         # :left_key :: foreign key in join table that points to current model's
         #              primary key, as a symbol. Defaults to :"#{self.name.underscore}_id".
         #              Can use an array of symbols for a composite key association.
@@ -2045,7 +2079,7 @@ module Sequel
             raise(Error, "mismatched number of right keys: #{rcks.inspect} vs #{rcpks.inspect}") unless rcks.length == rcpks.length
           end
           opts[:uses_left_composite_keys] = lcks.length > 1
-          opts[:uses_right_composite_keys] = rcks.length > 1
+          uses_rcks = opts[:uses_right_composite_keys] = rcks.length > 1
           opts[:cartesian_product_number] ||= one_through_one ? 0 : 1
           join_table = (opts[:join_table] ||= opts.default_join_table)
           opts[:left_key_alias] ||= opts.default_associated_key_alias
@@ -2054,8 +2088,75 @@ module Sequel
             opts[:after_load] ||= []
             opts[:after_load].unshift(:array_uniq!)
           end
-          opts[:dataset] ||= opts.association_dataset_proc
-          opts[:eager_loader] ||= opts.method(:default_eager_loader)
+          if join_table_db = opts[:join_table_db]
+            opts[:use_placeholder_loader] = false
+            opts[:allow_eager_graph] = false
+            opts[:allow_filtering_by] = false
+            opts[:eager_limit_strategy] = nil
+            join_table_ds = join_table_db.from(join_table)
+            opts[:dataset] ||= proc do |r|
+              vals = join_table_ds.where(lcks.zip(lcpks.map{|k| get_column_value(k)})).select_map(right)
+              ds = r.associated_dataset.where(opts.right_primary_key => vals)
+              if uses_rcks
+                vals.delete_if{|v| v.any?(&:nil?)}
+              else
+                vals.delete(nil)
+              end
+              ds = ds.clone(:no_results=>true) if vals.empty?
+              ds
+            end
+            opts[:eager_loader] ||= proc do |eo|
+              h = eo[:id_map]
+              assign_singular = opts.assign_singular?
+              rpk = opts.right_primary_key
+              name = opts[:name]
+
+              join_map = join_table_ds.where(left=>h.keys).select_hash_groups(right, left)
+
+              if uses_rcks
+                join_map.delete_if{|v,| v.any?(&:nil?)}
+              else
+                join_map.delete(nil)
+              end
+
+              eo = Hash[eo]
+
+              if join_map.empty?
+                eo[:no_results] = true
+              else
+                join_map.each_value do |vs|
+                  vs.replace(vs.flat_map{|v| h[v]})
+                  vs.uniq!
+                end
+
+                eo[:loader] = false
+                eo[:right_keys] = join_map.keys
+              end
+
+              opts[:model].eager_load_results(opts, eo) do |assoc_record|
+                rpkv = if uses_rcks
+                  assoc_record.values.values_at(*rpk)
+                else
+                  assoc_record.values[rpk]
+                end
+
+                objects = join_map[rpkv]
+
+                if assign_singular
+                  objects.each do |object|
+                    object.associations[name] ||= assoc_record
+                  end
+                else
+                  objects.each do |object|
+                    object.associations[name].push(assoc_record)
+                  end
+                end
+              end
+            end
+          else
+            opts[:dataset] ||= opts.association_dataset_proc
+            opts[:eager_loader] ||= opts.method(:default_eager_loader)
+          end
           
           join_type = opts[:graph_join_type]
           select = opts[:graph_select]
@@ -2417,7 +2518,7 @@ module Sequel
 
         # Dataset for the join table of the given many to many association reflection
         def _join_table_dataset(opts)
-          ds = model.db.from(opts.join_table_source)
+          ds = (opts[:join_table_db] || model.db).from(opts.join_table_source)
           opts[:join_table_block] ? opts[:join_table_block].call(ds) : ds
         end
 
@@ -2438,7 +2539,12 @@ module Sequel
           if loader = _associated_object_loader(opts, dynamic_opts)
             loader.all(*opts.predicate_key_values(self))
           else
-            _associated_dataset(opts, dynamic_opts).all
+            ds = _associated_dataset(opts, dynamic_opts)
+            if ds.opts[:no_results]
+              []
+            else
+              ds.all
+            end
           end
         end
 
@@ -2899,6 +3005,8 @@ module Sequel
               (multiple = ((op == :IN || op == :'NOT IN') && ((is_ds = r.is_a?(Sequel::Dataset)) || (r.respond_to?(:all?) && r.all?{|x| x.is_a?(Sequel::Model)})))))
             l = args[0]
             if ar = model.association_reflections[l]
+              raise Error, "filtering by associations is not allowed for #{ar.inspect}" if ar[:allow_filtering_by] == false
+
               if multiple
                 klass = ar.associated_class
                 if is_ds
@@ -3394,7 +3502,7 @@ module Sequel
         # Allow associations that are eagerly graphed to be specified as an SQL::AliasedExpression, for
         # per-call determining of the alias base.
         def eager_graph_check_association(model, association)
-          if association.is_a?(SQL::AliasedExpression)
+          reflection = if association.is_a?(SQL::AliasedExpression)
             expr = association.expression
             if expr.is_a?(SQL::Identifier)
               expr = expr.value
@@ -3403,10 +3511,17 @@ module Sequel
               end
             end
 
-            SQL::AliasedExpression.new(check_association(model, expr), association.alias || expr, association.columns)
+            check_reflection = check_association(model, expr)
+            SQL::AliasedExpression.new(check_reflection, association.alias || expr, association.columns)
           else
-            check_association(model, association)
+            check_reflection = check_association(model, association)
+          end
+
+          if check_reflection && check_reflection[:allow_eager_graph] == false
+            raise Error, "eager_graph not allowed for #{reflection.inspect}"
           end
+
+          reflection
         end
       
         # The EagerGraphLoader instance used for converting eager_graph results.

data/lib/sequel/plugins/many_through_many.rb

@@ -123,16 +123,25 @@ module Sequel
           nil
         end
 
+        # Whether a separate query should be used for each join table.
+        def separate_query_per_table?
+          self[:separate_query_per_table]
+        end
+
         private
 
         def _associated_dataset
           ds = associated_class
-          (reverse_edges + [final_reverse_edge]).each do |t|
-            h = {:qualify=>:deep}
-            if t[:alias] != t[:table]
-              h[:table_alias] = t[:alias]
+          if separate_query_per_table?
+            ds = ds.dataset
+          else
+            (reverse_edges + [final_reverse_edge]).each do |t|
+              h = {:qualify=>:deep}
+              if t[:alias] != t[:table]
+                h[:table_alias] = t[:alias]
+              end
+              ds = ds.join(t[:table], Array(t[:left]).zip(Array(t[:right])), h)
             end
-            ds = ds.join(t[:table], Array(t[:left]).zip(Array(t[:right])), h)
           end
           ds
         end
@@ -208,6 +217,7 @@ module Sequel
         #            :right (last array element) :: The key joining the table to the next table. Can use an
         #                                           array of symbols for a composite key association.
         #            If a hash is provided, the following keys are respected when using eager_graph:
+        #            :db :: The Database containing the table.  This changes lookup to use a separate query for each join table.
         #            :block :: A proc to use as the block argument to join.
         #            :conditions :: Extra conditions to add to the JOIN ON clause.  Must be a hash or array of two pairs.
         #            :join_type :: The join type to use for the join, defaults to :left_outer.
@@ -233,32 +243,121 @@ module Sequel
             opts[:after_load].unshift(:array_uniq!)
           end
           opts[:cartesian_product_number] ||= one_through_many ? 0 : 2
-          opts[:through] = opts[:through].map do |e|
+          separate_query_per_table = false
+          through = opts[:through] = opts[:through].map do |e|
             case e
             when Array
               raise(Error, "array elements of the through option/argument for many_through_many associations must have at least three elements") unless e.length == 3
               {:table=>e[0], :left=>e[1], :right=>e[2]}
             when Hash
               raise(Error, "hash elements of the through option/argument for many_through_many associations must contain :table, :left, and :right keys") unless e[:table] && e[:left] && e[:right]
+              separate_query_per_table = true if e[:db]
               e
             else
               raise(Error, "the through option/argument for many_through_many associations must be an enumerable of arrays or hashes")
             end
           end
+          opts[:separate_query_per_table] = separate_query_per_table
 
           left_key = opts[:left_key] = opts[:through].first[:left]
           opts[:left_keys] = Array(left_key)
-          opts[:uses_left_composite_keys] = left_key.is_a?(Array)
+          uses_lcks = opts[:uses_left_composite_keys] = left_key.is_a?(Array)
           left_pk = (opts[:left_primary_key] ||= self.primary_key)
           raise(Error, "no primary key specified for #{inspect}") unless left_pk
           opts[:eager_loader_key] = left_pk unless opts.has_key?(:eager_loader_key)
           opts[:left_primary_keys] = Array(left_pk)
           lpkc = opts[:left_primary_key_column] ||= left_pk
           lpkcs = opts[:left_primary_key_columns] ||= Array(lpkc)
-          opts[:dataset] ||= opts.association_dataset_proc
 
           opts[:left_key_alias] ||= opts.default_associated_key_alias
-          opts[:eager_loader] ||= opts.method(:default_eager_loader)
+          if separate_query_per_table
+            opts[:use_placeholder_loader] = false
+            opts[:allow_eager_graph] = false
+            opts[:allow_filtering_by] = false
+            opts[:eager_limit_strategy] = nil
+
+            opts[:dataset] ||= proc do |r|
+              def_db = r.associated_class.db
+              vals = uses_lcks ? [lpkcs.map{|k| get_column_value(k)}] : get_column_value(left_pk)
+
+              has_results = through.each do |edge|
+                ds = (edge[:db] || def_db).from(edge[:table]).where(edge[:left]=>vals)
+                ds = ds.where(edge[:conditions]) if edge[:conditions]
+                right = edge[:right]
+                vals = ds.select_map(right)
+                if right.is_a?(Array)
+                  vals.delete_if{|v| v.any?(&:nil?)}
+                else
+                  vals.delete(nil)
+                end
+                break if vals.empty?
+              end
+
+              ds = r.associated_dataset.where(opts.right_primary_key=>vals)
+              ds = ds.clone(:no_results=>true) unless has_results
+              ds
+            end
+            opts[:eager_loader] ||= proc do |eo|
+              h = eo[:id_map]
+              assign_singular = opts.assign_singular?
+              uses_rcks = opts.right_primary_key.is_a?(Array)
+              rpk = uses_rcks ? opts.right_primary_keys : opts.right_primary_key
+              name = opts[:name]
+              def_db = opts.associated_class.db
+              join_map = h
+
+              run_query = through.each do |edge|
+                ds = (edge[:db] || def_db).from(edge[:table])
+                ds = ds.where(edge[:conditions]) if edge[:conditions]
+                left = edge[:left]
+                right = edge[:right]
+                prev_map = join_map
+                join_map = ds.where(left=>join_map.keys).select_hash_groups(right, left)
+                if right.is_a?(Array)
+                  join_map.delete_if{|v,| v.any?(&:nil?)}
+                else
+                  join_map.delete(nil)
+                end
+                break if join_map.empty?
+                join_map.each_value do |vs|
+                  vs.replace(vs.flat_map{|v| prev_map[v]})
+                  vs.uniq!
+                end
+              end
+
+              eo = Hash[eo]
+
+              if run_query
+                eo[:loader] = false
+                eo[:right_keys] = join_map.keys
+              else
+                eo[:no_results] = true
+              end
+
+              opts[:model].eager_load_results(opts, eo) do |assoc_record|
+                rpkv = if uses_rcks
+                  assoc_record.values.values_at(*rpk)
+                else
+                  assoc_record.values[rpk]
+                end
+
+                objects = join_map[rpkv]
+
+                if assign_singular
+                  objects.each do |object|
+                    object.associations[name] ||= assoc_record
+                  end
+                else
+                  objects.each do |object|
+                    object.associations[name].push(assoc_record)
+                  end
+                end
+              end
+            end
+          else
+            opts[:dataset] ||= opts.association_dataset_proc
+            opts[:eager_loader] ||= opts.method(:default_eager_loader)
+          end
 
           join_type = opts[:graph_join_type]
           select = opts[:graph_select]

data/lib/sequel/plugins/prepared_statements.rb

@@ -169,8 +169,17 @@ module Sequel
           end
 
           case type
-          when :insert, :insert_select, :update
+          when :insert, :update
             true
+          when :insert_select
+            # SQLite RETURNING support has a bug that doesn't allow for committing transactions
+            # when a prepared statement with RETURNING has been used on the connection:
+            #
+            #   SQLite3::BusyException: cannot commit transaction - SQL statements in progress: COMMIT
+            #
+            # Disabling usage of prepared statements for insert_select on SQLite seems to be the
+            # simplest way to workaround the problem.
+            db.database_type != :sqlite
           # :nocov:
           when :delete, :refresh
             Sequel::Deprecation.deprecate("The :delete and :refresh prepared statement types", "There should be no need to check if these types are supported")

data/lib/sequel/plugins/timestamps.rb

@@ -19,7 +19,7 @@ module Sequel
     #
     #   # Timestamp Artist instances, forcing an overwrite of the create
     #   # timestamp, and setting the update timestamp when creating
-    #   Album.plugin :timestamps, force: true, update_on_create: true
+    #   Artist.plugin :timestamps, force: true, update_on_create: true
     module Timestamps
       # Configure the plugin by setting the available options.  Note that
       # if this method is run more than once, previous settings are ignored,

data/lib/sequel/plugins/unused_associations.rb

@@ -214,11 +214,11 @@ module Sequel
     # reported by this plugin.
     #
     # This plugin only considers the public instance methods the
-    # association defines to determine if it was used.  If an
-    # association is used in a way that does not call an instance
-    # method (such as only using the association with the
-    # dataset_associations plugin), then it would show up as unused
-    # by this plugin.
+    # association defines, and direct access to the related
+    # association reflection via Sequel::Model.association_reflection
+    # to determine if the association was used.  If the association
+    # metadata was accessed another way, it's possible this plugin
+    # will show the association as unused.
     #
     # As this relies on the method coverage added in Ruby 2.5, it does
     # not work on older versions of Ruby.  It also does not work on
@@ -266,6 +266,18 @@ module Sequel
         # unused_associations only on the class that is loading the plugin.
         Plugins.inherited_instance_variables(self, :@unused_associations_data=>nil)
 
+        # Synchronize access to the used association reflections.
+        def used_association_reflections
+          Sequel.synchronize{@used_association_reflections ||= {}}
+        end
+
+        # Record access to association reflections to determine which associations are not used.
+        def association_reflection(association)
+          uar = used_association_reflections
+          Sequel.synchronize{uar[association] ||= true}
+          super
+        end
+
         # If modifying associations, and this association is marked as not used,
         # and the association does not include the specific :is_used option,
         # skip defining the association.
@@ -277,6 +289,12 @@ module Sequel
           super
         end
 
+        # Setup the used_association_reflections storage before freezing
+        def freeze
+          used_association_reflections
+          super
+        end
+
         # Parse the coverage result, and return the coverage data for the
         # associations for descendants of this class. If the plugin
         # uses the :coverage_file option, the existing coverage file will be loaded
@@ -298,7 +316,7 @@ module Sequel
           ([self] + descendents).each do |sc|
             next if sc.associations.empty? || !sc.name
             module_mapping[sc.send(:overridable_methods_module)] = sc
-            coverage_data[sc.name] ||= {}
+            coverage_data[sc.name] ||= {''=>sc.used_association_reflections.keys.map(&:to_s).sort}
           end
 
           coverage_result.each do |file, coverage|
@@ -331,10 +349,9 @@ module Sequel
 
           ([self] + descendents).each do |sc|
             next unless cov_data = coverage_data[sc.name]
+            reflection_data = cov_data[''] || []
 
-            sc.associations.each do |assoc|
-              ref = sc.association_reflection(assoc)
-
+            sc.association_reflections.each do |assoc, ref|
               # Only report associations for the class they are defined in
               next unless ref[:model] == sc
 
@@ -343,6 +360,9 @@ module Sequel
               next if ref[:methods_module]
 
               info = {}
+              if reflection_data.include?(assoc.to_s)
+                info[:used] = [:reflection]
+              end
 
               _update_association_coverage_info(info, cov_data, ref.dataset_method, :dataset_method)
               _update_association_coverage_info(info, cov_data, ref.association_method, :association_method)

data/lib/sequel/version.rb

@@ -6,7 +6,7 @@ module Sequel
 
   # The minor version of Sequel.  Bumped for every non-patch level
   # release, generally around once a month.
-  MINOR = 46
+  MINOR = 47
 
   # The tiny version of Sequel.  Usually 0, only bumped for bugfix
   # releases that fix regressions from previous versions.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequel
 version: !ruby/object:Gem::Version
-  version: 5.46.0
+  version: 5.47.0
 platform: ruby
 authors:
 - Jeremy Evans
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-01 00:00:00.000000000 Z
+date: 2021-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -190,6 +190,7 @@ extra_rdoc_files:
 - doc/release_notes/5.44.0.txt
 - doc/release_notes/5.45.0.txt
 - doc/release_notes/5.46.0.txt
+- doc/release_notes/5.47.0.txt
 - doc/release_notes/5.5.0.txt
 - doc/release_notes/5.6.0.txt
 - doc/release_notes/5.7.0.txt
@@ -264,6 +265,7 @@ files:
 - doc/release_notes/5.44.0.txt
 - doc/release_notes/5.45.0.txt
 - doc/release_notes/5.46.0.txt
+- doc/release_notes/5.47.0.txt
 - doc/release_notes/5.5.0.txt
 - doc/release_notes/5.6.0.txt
 - doc/release_notes/5.7.0.txt
@@ -575,7 +577,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: The Database Toolkit for Ruby