sequel 3.44.0 → 3.45.0

data/CHANGELOG

@@ -1,3 +1,47 @@
+=== 3.45.0 (2013-03-01)
+
+* Remove bad model typecasting of money type on PostgreSQL (jeremyevans) (#624)
+
+* Use simplecov instead of rcov for coverage testing on 1.9+ (jeremyevans)
+
+* Make the Database#quote_identifier method public (jeremyevans)
+
+* Make PostgreSQL metadata parsing handle tables with the same name in multiple schemas (jeremyevans)
+
+* Switch query extension to use a proxy instead of Object#extend (chanks, jeremyevans)
+
+* Remove Dataset#def_mutiation_method instance method (jeremyevans)
+
+* Make foreign key parsing on MySQL not pick up foreign keys in other databases (jeremyevans)
+
+* Allow per-instance overrides of Postgres.force_standard_strings and .client_min_messages (jeremyevans) (#618)
+
+* Add Sequel.tzinfo_disambiguator= to the named_timezones plugin for automatically handling TZInfo::AmbiguousTime exceptions (jeremyevans) (#616)
+
+* Add Dataset#escape_like, for escaping LIKE metacharacters (jeremyevans) (#614)
+
+* The LIKE operators now use an explicit ESCAPE '\' clause for similar behavior across databases (jeremyevans)
+
+* Make Database#tables and #views accept a :qualify option on PostgreSQL to return qualified identifiers (jeremyevans)
+
+* Make json_serializer and xml_serializer plugins secure by default (jeremyevans)
+
+* Address JSON.parse vulnerabilities (jeremyevans)
+
+* Fix Dataset#from_self! to no longer create a self-referential dataset (jeremyevans)
+
+* Use SQLSTATE or database error codes if available instead of regexp parsing for more specific DatabaseErrors (jeremyevans)
+
+* Add unlimited_update plugin to work around MySQL warning in replicated environments (jeremyevans)
+
+* Add the :retry_on and :num_retries transaction options for automatically retrying transactions (jeremyevans)
+
+* Raise serialization failures/deadlocks as Sequel::SerializationFailure exceptions (jeremyevans)
+
+* Support transaction isolation levels on Oracle and DB2 (jeremyevans)
+
+* Support transaction isolation levels when using the JDBC transaction support (jeremyevans)
+
 === 3.44.0 (2013-02-04)
 
 * Speedup mysql2 adapter with identifier output method fetch speed by up to 50% (jeremyevans)

data/Rakefile

@@ -125,10 +125,18 @@ begin
 
   spec_with_cov = lambda do |name, files, d, &b|
     spec.call(name, files, d)
-    t = spec.call("#{name}_cov", files, "#{d} with coverage")
-    t.rcov = true
-    t.rcov_opts = File.read("spec/rcov.opts").split("\n")
-    b.call(t) if b
+    if RUBY_VERSION < '1.9'
+      t = spec.call("#{name}_cov", files, "#{d} with coverage")
+      t.rcov = true
+      t.rcov_opts = File.read("spec/rcov.opts").split("\n")
+      b.call(t) if b
+    else
+      desc "#{d} with coverage"
+      task "#{name}_cov" do
+        ENV['COVERAGE'] = '1'
+        Rake::Task[name].invoke
+      end
+    end
     t
   end
 

data/doc/reflection.rdoc

@@ -99,12 +99,12 @@ You can get an array of association symbols with Model.associations:
 
 You can get the association reflection for a single association via the Model.association_reflection.  Association reflections are subclasses of hash, for ease of use and introspection (and backwards compatibility):
 
-  Model.association_reflection(:association1) # {:name=>:association1, :type=>:many_to_one, :model=>Model, :associated_class=>OtherModel, ...}
+  Model.association_reflection(:association1) # {:name=>:association1, :type=>:many_to_one, :model=>Model, ...}
 
 You can get an array of all association reflections via Model.all_association_reflections:
 
-  Model.all_association_reflections # [{:name=>:association1, :type=>:many_to_one, :model=>Model, :associated_class=>OtherModel, ...}, ...]
+  Model.all_association_reflections # [{:name=>:association1, :type=>:many_to_one, :model=>Model, ...}, ...]
 
 Finally, you can get a hash of association reflections via Model.association_reflections:
 
-  Model.association_reflections # {:association1=>{:name=>:association1, :type=>:many_to_one, :model=>Model, :associated_class=>OtherModel, ...}, ...}
+  Model.association_reflections # {:association1=>{:name=>:association1, :type=>:many_to_one, :model=>Model, ...}, ...}

data/doc/release_notes/3.45.0.txt

@@ -0,0 +1,179 @@
+= New Features
+
+* Database#transaction now recognizes a :retry_on option, which
+  should contain an exception class or array of exception classes.
+  If the transaction raises one of the given exceptions, Sequel
+  will automatically retry the transaction block.  It's a bad idea to
+  use this option if the transaction block is not idempotent.
+
+  By default, Sequel only retries the block 5 times by default,
+  to protect against infinite looping.  You can change the number
+  of retries with the :num_retries option.
+
+  Users of the :disconnect=>:retry option are encouraged to switch
+  to :retry_on=>Sequel::DatabaseDisconnectError.
+
+* Dataset#escape_like has been added for escaping LIKE
+  metacharacters.  This is designed for the case where part of
+  the LIKE pattern is based on user input that should not treat the
+  metacharacters specially.
+
+* Serialization failures/deadlocks are now raised as
+  Sequel::SerializationFailure exception instances.  This exception
+  class is a good candidate for the transaction :retry_on option.
+
+* On PostgreSQL, you can now provide the :force_standard_strings
+  and :client_min_messages Database options to override the defaults
+  on a per-instance basis.
+
+* On PostgreSQL, Database#tables and #views now recognizes a
+  :qualify option, which if true will return qualified identifiers
+  instead of plain symbols.
+
+* Transaction isolation levels are now supported on Oracle, DB2,
+  and all jdbc subadapters using the JDBC transaction support.
+
+* Dataset.def_mutation_method now accepts a :module option for
+  the module in which to define the methods (defaulting to self).
+
+* An unlimited_update plugin has been added.  It's sole purpose is to
+  eliminate a MySQL warning in replicated environments, since by
+  default Sequel::Model uses a LIMIT clause when updating on MySQL.
+
+* The named_timezones extension now adds a
+  Sequel.tzinfo_disambiguator accessor to automatically handle
+  TZInfo::AmbiguousTime exceptions.  This should be a callable object
+  that accepts two arguments, a DateTime instance and an array of
+  timezone periods, and returns the timezone period to use.
+
+= Other Improvements
+
+* Sequel now handles JSON securely, specifying the
+  :create_additions=>false option when using JSON.parse.  If you
+  really want to get the old vulnerable behavior back, override
+  Sequel.parse_json.
+
+* The json_serializer and xml_serializer plugins are now secure
+  by default.  Before, the default behavior of these plugins
+  allowed for round tripping, such that:
+
+    Album.from_xml(album.to_xml) == album
+
+  Unfortunately, that requires that the deserialization allow
+  the setting of any column.  Since the plugins also handle
+  associations, you could also set any column in any associated
+  object, even cascading to associated objects of those objects.
+
+  The new default behavior only allows deserialization to set
+  the same columns that mass-assignment would set, and not to
+  handle associated objects at all by default.  The following
+  additional options are supported:
+
+    :fields :: The specific fields to set (this was already supported
+               by the json_serializer plugin).
+    :associations :: The specific associations to handle.
+    :all_columns :: The previous behavior of setting all columns.
+    :all_associations :: The previous behavior of setting all
+                         associations.
+
+  Since JSON parsing no longer deserializes into arbitrary ruby
+  instances, from_json and array_from_json class methods have been
+  added to the json_serializer plugin, for deserializing into model
+  instances.  These mirror the from_xml and array_from_xml class
+  methods in the xml_serializer plugin.
+
+  Note that the :all_columns and :all_associations methods were
+  only added to make backwards compatibility easier.  It is
+  likely they will be removed in Sequel 4, along with the
+  json_create class method.
+
+* Sequel now attempts to use database specific error codes or
+  SQLState codes instead of regexp parsing to determine if a more
+  specific DatabaseError subclass should be used.  This should make
+  error handling faster and more robust.
+
+* Sequel now uses ESCAPE '\' when using LIKE, for similar behavior
+  across databases.  Previously, no ESCAPE clause was used, so
+  behavior differed across databases, with most not using escaping,
+  and PostgreSQL, MySQL, and H2 defaulting to backslash as the escape
+  character.
+
+* The query extension has been reimplemented and now uses a proxy
+  object instead of Object#extend.
+
+* The :pool_timeout Database option now supports fractional seconds.
+
+* Database#quote_identifier is now a public method.
+
+* Metadata parsing (schema, indexes, foreign_key_list) on PostgreSQL
+  now correctly handles the case where an unqualified table name is
+  used and tables with that name exist in multiple schemas.  It now
+  picks the first matching table in the schema_search_path, instead of
+  failing or returning results from all tables.
+
+* Sequel::Model instances no longer attempt to typecast the money
+  type on PostgreSQL, since the previous typecast didn't work
+  correctly, and correct typecasting is locale-dependent.
+
+* Sequel no longer picks up foreign keys for tables in other
+  databases when using Database#foreign_key_list on MySQL.
+
+* A warning when using the mysql2 3.12 beta has been eliminated.
+
+* A warning has been eliminated when using the jdbc/oracle adapter
+  on JRuby 1.7.
+
+* Sequel's ilike emulation should now work by default on databases
+  without specific syntax support.
+
+* Dataset#from_self! no longer creates a self referential dataset.
+
+* Coverage testing now uses simplecov instead of rcov on ruby 1.9+.
+
+= Backwards Compatibility
+
+* The switch to using JSON.parse :create_additions=>false means
+  that if your app expected JSON to deserialize into arbitrary
+  ruby objects, it is probably broken.  You should update your
+  application code to manually convert the deserialized hashes
+  into the ruby objects you want.
+
+  Note that it's not just this new version of Sequel that will
+  cause that, older versions of Sequel will break in the same
+  way if you update your JSON library to a version that is not
+  vulnerable by default.
+
+  This potentially affects the pg_json extension and serialization
+  plugin if you were expecting the JSON stored in the database
+  to be deserialized into arbitrary ruby objects.
+
+  See the json_serializer/xml_serializer changes mentioned in
+  the Other Improvements section.
+
+* The reimplemented query extension is not completely backwards
+  compatible.  For example, inside a query block, self refers to the
+  proxy object instead of a dataset, and calling methods that return
+  rows no longer raises an exception.
+
+* The metadata parsing methods on PostgreSQL no longer work with
+  unqualified tables where the table is not in the schema search
+  path.  This makes metadata parsing consistent with how datasets
+  operate.  For tables outside the schema search path, you must
+  qualify it before use now.
+
+  Additionally, using a nonexistent table name will raise an
+  exception instead of returning empty results in some cases.
+
+* The Dataset#def_mutation_method instance method has been removed.
+  This method added mutation methods directly on the dataset instance,
+  which is generally not desired.  Using the def_mutation_method class
+  method with the :module option is now the recommended approach.
+
+* The switch to using ESCAPE for LIKE characters is backwards
+  incompatible on databases that don't use escaping by default, 
+  when backslash is used in a LIKE pattern as a regular character.
+  Now you have to double the backslash in the pattern.
+
+* Database#database_error_regexps private method now can return any
+  enumerable yielding regexp/exception class pairs, it is no longer
+  specified to return a hash.

data/doc/schema_modification.rdoc

@@ -532,7 +532,7 @@ keys before the tables containing the primary keys they reference.
 === <tt>drop_table?</tt>
 
 <tt>drop_table?</tt> is similar to drop_table, except that it only drops
-the table if the table does not already exist.  On some databases, it uses
+the table if the table already exists.  On some databases, it uses
 <tt>IF NOT EXISTS</tt>, on others it does a separate query to check for
 existence.
 

data/doc/transactions.rdoc

@@ -135,3 +135,26 @@ The SQL standard supports 4 isolation levels: READ UNCOMMITTED, READ COMMITTED,
     # SET TRANSACTION ISOLATION LEVEL SERIALIZABLE
     DB[:foo].insert(1) # INSERT
   end # COMMIT
+
+== Automatically Restarting Transactions
+
+Sequel offers the ability to automatically restart transactions if specific types of errors are detected.  For example, if you want to automatically restart a transaction if a serialization failure is detected:
+
+  DB.transaction(:isolation => :serializable, :retry_on=>[Sequel::SerializationFailure]) do
+    ModelClass.find_or_create(:name=>'Foo')
+  end
+
+At the serializable transaction isolation level, find_or_create may raises a Sequel::SerializationFailure exception if multiple threads simultaneously run that code.  With the :retry_on option set, the transaction will be automatically retried until it succeeds.
+
+Note that automatic retrying should not be used unless the entire transaction
+block is idempotent, as otherwise it can cause non-idempotent
+behavior to execute multiple times.  For example, with the following code:
+
+  DB.transaction(:isolation => :serializable, :retry_on=>[Sequel::SerializationFailure]) do
+    logger.info 'Ensuring existence of ModelClass with name Foo'
+    ModelClass.find_or_create(:name=>'Foo')
+  end
+
+The logger.info method will be called multiple times if there is a serialization failure.
+
+The :num_retries option can be used to set the maxmimum number of times to retry.  It is set to 5 times by default.

data/lib/sequel/adapters/db2.rb

@@ -122,6 +122,7 @@ module Sequel
 
       def begin_transaction(conn, opts={})
         log_yield(TRANSACTION_BEGIN){DB2CLI.SQLSetConnectAttr(conn, DB2CLI::SQL_ATTR_AUTOCOMMIT, DB2CLI::SQL_AUTOCOMMIT_OFF)}
+        set_transaction_isolation(conn, opts)
       end
 
       def remove_transaction(conn, committed)

data/lib/sequel/adapters/ibmdb.rb

@@ -35,6 +35,12 @@ module Sequel
 
       # Error class for exceptions raised by the connection.
       class Error < StandardError
+        attr_reader :sqlstate
+
+        def initialize(message, sqlstate)
+          @sqlstate = sqlstate
+          super(message)
+        end
       end
 
       # Create the underlying IBM_DB connection.
@@ -69,11 +75,16 @@ module Sequel
         IBM_DB.getErrormsg(@conn, IBM_DB::DB_CONN)
       end
 
+      # Return the related error message for the connection.
+      def error_sqlstate
+        IBM_DB.getErrorstate(@conn, IBM_DB::DB_CONN)
+      end
+
       # Execute the given SQL on the database, and return a Statement instance
       # holding the results.
       def execute(sql)
         stmt = IBM_DB.exec(@conn, sql)
-        raise Error, error_msg unless stmt
+        raise Error.new(error_msg, error_sqlstate) unless stmt
         Statement.new(stmt)
       end
 
@@ -83,7 +94,7 @@ module Sequel
         stmt = @prepared_statements[ps_name].last
         res = stmt.execute(*values)
         unless res
-          raise Error, "Error executing statement #{ps_name}: #{error_msg}"
+          raise Error.new("Error executing statement #{ps_name}: #{error_msg}", error_sqlstate)
         end
         stmt
       end
@@ -98,7 +109,7 @@ module Sequel
         else
           err = error_msg
           err = "Error preparing #{ps_name} with SQL: #{sql}" if error_msg.nil? || error_msg.empty?
-          raise Error, err
+          raise Error.new(err, error_sqlstate)
         end
       end
 
@@ -301,6 +312,7 @@ module Sequel
       # So starting a transaction just turns autocommit off.
       def begin_transaction(conn, opts={})
         log_yield(TRANSACTION_BEGIN){conn.autocommit = false}
+        set_transaction_isolation(conn, opts)
       end
 
       # This commits transaction in progress on the
@@ -313,6 +325,10 @@ module Sequel
         [Connection::Error]
       end
 
+      def database_exception_sqlstate(exception, opts)
+        exception.sqlstate
+      end
+
       # Don't convert smallint to boolean for the metadata
       # dataset, since the DB2 metadata does not use
       # boolean columns, and some smallint columns are

data/lib/sequel/adapters/jdbc.rb

@@ -345,6 +345,21 @@ module Sequel
         [NativeException]
       end
 
+      def database_exception_sqlstate(exception, opts)
+        if database_exception_use_sqlstates?
+          while exception.respond_to?(:cause)
+            exception = exception.cause
+            return exception.getSQLState if exception.respond_to?(:getSQLState)
+          end
+        end
+        nil
+      end
+
+      # Whether the JDBC subadapter should use SQL states for exception handling, true by default.
+      def database_exception_use_sqlstates?
+        true
+      end
+
       # Raise a disconnect error if the SQL state of the cause of the exception indicates so.
       def disconnect_error?(exception, opts)
         cause = exception.respond_to?(:cause) ? exception.cause : exception

data/lib/sequel/adapters/jdbc/derby.rb

@@ -109,6 +109,7 @@ module Sequel
           /violation of foreign key constraint/ => ForeignKeyConstraintViolation,
           /The check constraint .+ was violated/ => CheckConstraintViolation,
           /cannot accept a NULL value/ => NotNullConstraintViolation,
+          /A lock could not be obtained due to a deadlock/ => SerializationFailure,
         }.freeze
         def database_error_regexps
           DATABASE_ERROR_REGEXPS
@@ -208,13 +209,8 @@ module Sequel
           end
         end
 
-        # Handle Derby specific LIKE, extract, and some bitwise compliment support.
         def complex_expression_sql_append(sql, op, args)
           case op
-          when :ILIKE
-            super(sql, :LIKE, [SQL::Function.new(:upper, args.at(0)), SQL::Function.new(:upper, args.at(1))])
-          when :"NOT ILIKE"
-            super(sql, :"NOT LIKE", [SQL::Function.new(:upper, args.at(0)), SQL::Function.new(:upper, args.at(1))])
           when :%
             sql << complex_expression_arg_pairs(args){|a, b| "MOD(#{literal(a)}, #{literal(b)})"}
           when :&, :|, :^, :<<, :>>

data/lib/sequel/adapters/jdbc/h2.rb

@@ -101,6 +101,7 @@ module Sequel
           /Referential integrity constraint violation/ => ForeignKeyConstraintViolation,
           /Check constraint violation/ => CheckConstraintViolation,
           /NULL not allowed for column/ => NotNullConstraintViolation,
+          /Deadlock detected\. The current transaction was rolled back\./ => SerializationFailure,
         }.freeze
         def database_error_regexps
           DATABASE_ERROR_REGEXPS

data/lib/sequel/adapters/jdbc/hsqldb.rb

@@ -57,6 +57,7 @@ module Sequel
           /integrity constraint violation: foreign key/ => ForeignKeyConstraintViolation,
           /integrity constraint violation: check constraint/ => CheckConstraintViolation,
           /integrity constraint violation: NOT NULL check constraint/ => NotNullConstraintViolation,
+          /serialization failure/ => SerializationFailure,
         }.freeze
         def database_error_regexps
           DATABASE_ERROR_REGEXPS
@@ -124,7 +125,7 @@ module Sequel
         def complex_expression_sql_append(sql, op, args)
           case op
           when :ILIKE, :"NOT ILIKE"
-            super(sql, (op == :ILIKE ? :LIKE : :"NOT LIKE"), [SQL::Function.new(:ucase, args.at(0)), SQL::Function.new(:ucase, args.at(1)) ])
+            super(sql, (op == :ILIKE ? :LIKE : :"NOT LIKE"), args.map{|v| SQL::Function.new(:ucase, v)})
           when :&, :|, :^
             op = BITWISE_METHOD_MAP[op]
             sql << complex_expression_arg_pairs(args){|a, b| literal(SQL::Function.new(op, a, b))}