sequel 3.0.0 → 3.1.0

data/lib/sequel/extensions/schema_dumper.rb

@@ -1,17 +1,29 @@
+# The schema_dumper extension supports dumping tables and indexes
+# in a Sequel::Migration format, so they can be restored on another
+# database (which can be the same type or a different type than
+# the current database).  The main interface is through
+# Sequel::Database#dump_schema_migration.
+
 module Sequel
   class Database
+    POSTGRES_DEFAULT_RE = /\A(?:B?('.*')::[^']+|\((-?\d+(?:\.\d+)?)\))\z/
+    MYSQL_TIMESTAMP_RE = /\ACURRENT_(?:DATE|TIMESTAMP)?\z/
+    STRING_DEFAULT_RE = /\A'(.*)'\z/
+    
     # Dump indexes for all tables as a migration.  This complements
-    # the :indexes=>false option to dump_schema_migration.
-    def dump_indexes_migration
+    # the :indexes=>false option to dump_schema_migration. Options:
+    # * :same_db - Create a dump for the same database type, so
+    #   don't ignore errors if the index statements fail.
+    def dump_indexes_migration(options={})
       ts = tables
       <<END_MIG
 Class.new(Sequel::Migration) do
   def up
-#{ts.map{|t| dump_table_indexes(t, :add_index)}.reject{|x| x == ''}.join("\n\n").gsub(/^/o, '    ')}
+#{ts.sort_by{|t| t.to_s}.map{|t| dump_table_indexes(t, :add_index, options)}.reject{|x| x == ''}.join("\n\n").gsub(/^/o, '    ')}
   end
   
   def down
-#{ts.map{|t| dump_table_indexes(t, :drop_index)}.reject{|x| x == ''}.join("\n\n").gsub(/^/o, '    ')}
+#{ts.sort_by{|t| t.to_s}.map{|t| dump_table_indexes(t, :drop_index, options)}.reject{|x| x == ''}.join("\n\n").gsub(/^/o, '    ')}
   end
 end
 END_MIG
@@ -31,11 +43,11 @@ END_MIG
       <<END_MIG
 Class.new(Sequel::Migration) do
   def up
-#{ts.map{|t| dump_table_schema(t, options)}.join("\n\n").gsub(/^/o, '    ')}
+#{ts.sort_by{|t| t.to_s}.map{|t| dump_table_schema(t, options)}.join("\n\n").gsub(/^/o, '    ')}
   end
   
   def down
-    drop_table(#{ts.inspect[1...-1]})
+    drop_table(#{ts.sort_by{|t| t.to_s}.inspect[1...-1]})
   end
 end
 END_MIG
@@ -56,23 +68,70 @@ END_MIG
         indexes.each{|iname, iopts| send(:index, iopts[:columns], im.call(table, iname, iopts))} if indexes
       end
       commands = [gen.dump_columns, gen.dump_constraints, gen.dump_indexes].reject{|x| x == ''}.join("\n\n")
-      "create_table(#{table.inspect}) do\n#{commands.gsub(/^/o, '  ')}\nend"
+      "create_table(#{table.inspect}#{', :ignore_index_errors=>true' if !options[:same_db] && options[:indexes] != false && indexes && !indexes.empty?}) do\n#{commands.gsub(/^/o, '  ')}\nend"
     end
 
     private
-
+    
     # Convert the given default, which should be a database specific string, into
-    # a ruby object.  If it can't be converted, return the string with the inspect
-    # method modified so that .lit is always appended after it.
-    def column_schema_to_ruby_default(default, type)
-      case default 
-      when /false/
-        false
-      when 'true'
-        true
-      when /\A\d+\z/
-        default.to_i
-      else
+    # a ruby object.
+    def column_schema_to_ruby_default(default, type, options)
+      return if default.nil?
+      orig_default = default
+      if database_type == :postgres and m = POSTGRES_DEFAULT_RE.match(default)
+        default = m[1] || m[2]
+      end
+      if [:string, :blob, :date, :datetime, :time].include?(type)
+        if database_type == :mysql
+          if [:date, :datetime, :time].include?(type) && MYSQL_TIMESTAMP_RE.match(default)
+            return column_schema_to_ruby_default_fallback(default, options)
+          end
+          orig_default = default = "'#{default.gsub("'", "''").gsub('\\', '\\\\')}'" 
+        end
+        if m = STRING_DEFAULT_RE.match(default)
+          default = m[1].gsub("''", "'")
+        else
+          return column_schema_to_ruby_default_fallback(default, options)
+        end
+      end
+      res = begin
+        case type
+        when :boolean
+          case default 
+          when /[f0]/i
+            false
+          when /[t1]/i
+            true
+          end
+        when :string
+          default
+        when :blob
+          Sequel::SQL::Blob.new(default)
+        when :integer
+          Integer(default)
+        when :float
+          Float(default)
+        when :date
+          Sequel.string_to_date(default)
+        when :datetime
+          DateTime.parse(default)
+        when :time
+          Sequel.string_to_time(default)
+        when :decimal
+          BigDecimal.new(default)
+        end
+      rescue
+        nil
+      end
+      res.nil? ? column_schema_to_ruby_default_fallback(orig_default, options) : res
+    end
+        
+    # If the database default can't be converted, return the string with the inspect
+    # method modified so that .lit is always appended after it, only if the
+    # :same_db option is used.
+    def column_schema_to_ruby_default_fallback(default, options)
+      if options[:same_db]
+        default = default.to_s
         def default.inspect
           "#{super}.lit"
         end
@@ -89,7 +148,8 @@ END_MIG
         col_opts = options[:same_db] ? {:type=>schema[:db_type]} : column_schema_to_ruby_type(schema)
         type = col_opts.delete(:type)
         col_opts.delete(:size) if col_opts[:size].nil?
-        col_opts[:default] = column_schema_to_ruby_default(schema[:default], type) if schema[:default]
+        default = column_schema_to_ruby_default(schema[:default], schema[:type], options) if schema[:default]
+        col_opts[:default] = default unless default.nil?
         col_opts[:null] = false if schema[:allow_null] == false
         [:column, name, type, col_opts]
       end
@@ -141,14 +201,14 @@ END_MIG
 
     # Return a string that containing add_index/drop_index method calls for
     # creating the index migration.
-    def dump_table_indexes(table, meth)
+    def dump_table_indexes(table, meth, options={})
       return '' unless respond_to?(:indexes)
       im = method(:index_to_generator_opts)
       indexes = indexes(table).sort_by{|k,v| k.to_s} 
       gen = Schema::Generator.new(self) do
         indexes.each{|iname, iopts| send(:index, iopts[:columns], im.call(table, iname, iopts))}
       end
-      gen.dump_indexes(meth=>table)
+      gen.dump_indexes(meth=>table, :ignore_errors=>!options[:same_db])
     end
 
     # Convert the parsed index information into options to the Generators index method. 
@@ -216,12 +276,13 @@ END_MIG
       #   can be called outside of a generator but inside a migration.
       #   The value of this option should be the table name to use.
       # * :drop_index - Same as add_index, but create drop_index statements.
+      # * :ignore_errors - Add the ignore_errors option to the outputted indexes
       def dump_indexes(options={})
         indexes.map do |c|
           c = c.dup
           cols = c.delete(:columns)
           if table = options[:add_index] || options[:drop_index]
-            "#{options[:drop_index] ? 'drop' : 'add'}_index #{table.inspect}, #{cols.inspect}#{opts_inspect(c)}"
+            "#{options[:drop_index] ? 'drop' : 'add'}_index #{table.inspect}, #{cols.inspect}#{', :ignore_errors=>true' if options[:ignore_errors]}#{opts_inspect(c)}"
           else
             "index #{cols.inspect}#{opts_inspect(c)}"
           end
@@ -231,7 +292,22 @@ END_MIG
       private
 
       def opts_inspect(opts)
-        ", #{opts.inspect[1...-1]}" if opts.length > 0
+        if opts[:default]
+          opts = opts.dup
+          de = case d = opts.delete(:default)
+          when BigDecimal, Sequel::SQL::Blob
+            "#{d.class.name}.new(#{d.to_s.inspect})"
+          when DateTime, Date
+            "#{d.class.name}.parse(#{d.to_s.inspect})"
+          when Time
+            "#{d.class.name}.parse(#{d.strftime('%H:%M:%S').inspect})"
+          else
+            d.inspect
+          end
+          ", :default=>#{de}#{", #{opts.inspect[1...-1]}" if opts.length > 0}"
+        else
+          ", #{opts.inspect[1...-1]}" if opts.length > 0
+        end
       end
     end
   end

data/lib/sequel/extensions/string_date_time.rb

@@ -1,7 +1,6 @@
-# This file contains the previous extensions to String for date/time
-# conversions.  These are provided mainly for backward compatibility,
-# Sequel now uses a module level method instead of extending string
-# to handle the internal conversions.
+# The string_date_time extension provides String instance methods
+# for converting the strings to a date (e.g. String#to_date), allowing
+# for backwards compatibility with legacy Sequel code.
 
 class String
   # Converts a string into a Date object.

data/lib/sequel/model.rb

@@ -67,7 +67,7 @@ module Sequel
       :@raise_on_save_failure=>nil, :@restricted_columns=>:dup, :@restrict_primary_key=>nil,
       :@simple_pk=>nil, :@simple_table=>nil, :@strict_param_setting=>nil,
       :@typecast_empty_string_to_nil=>nil, :@typecast_on_assignment=>nil,
-      :@raise_on_typecast_failure=>nil}
+      :@raise_on_typecast_failure=>nil, :@plugins=>:dup}
 
     # Regexp that determines if a method name is normal in the sense that
     # it could be called directly in ruby code without using send.  Used to
@@ -88,6 +88,7 @@ module Sequel
     @dataset_method_modules = []
     @dataset_methods = {}
     @overridable_methods_module = nil
+    @plugins = []
     @primary_key = :id
     @raise_on_save_failure = true
     @raise_on_typecast_failure = true

data/lib/sequel/model/associations.rb

@@ -57,6 +57,7 @@ module Sequel
           self[:class] ||= constantize(self[:class_name])
         end
       
+
         # Name symbol for the dataset association method
         def dataset_method
           :"#{self[:name]}_dataset"
@@ -72,6 +73,12 @@ module Sequel
           true
         end
     
+        # By default associations do not need to select a key in an associated table
+        # to eagerly load.
+        def eager_loading_use_associated_key?
+          false
+        end
+
         # Whether to eagerly graph a lazy dataset, true by default.  If this
         # is false, the association won't respect the :eager_graph option
         # when loading the association for a single record.
@@ -96,7 +103,7 @@ module Sequel
           r_type = reciprocal_type
           key = self[:key]
           associated_class.all_association_reflections.each do |assoc_reflect|
-            if assoc_reflect[:type] == r_type && assoc_reflect[:key] == key
+            if assoc_reflect[:type] == r_type && assoc_reflect[:key] == key && assoc_reflect.associated_class == self[:model]
               return self[:reciprocal] = assoc_reflect[:name]
             end
           end
@@ -223,11 +230,31 @@ module Sequel
       class ManyToManyAssociationReflection < AssociationReflection
         ASSOCIATION_TYPES[:many_to_many] = self
     
+        # The alias to use for the associated key when eagerly loading
+        def associated_key_alias
+          self[:left_key_alias]
+        end
+
+        # The column to use for the associated key when eagerly loading
+        def associated_key_column
+          self[:left_key]
+        end
+
+        # The table containing the column to use for the associated key when eagerly loading
+        def associated_key_table
+          self[:join_table]
+        end
+
+        # The default associated key alias
+        def default_associated_key_alias
+          :x_foreign_key_x
+        end
+      
         # Default name symbol for the join table.
         def default_join_table
           [self[:class_name], self[:model].name].map{|i| underscore(pluralize(demodulize(i)))}.sort.join('_').to_sym
         end
-      
+
         # Default foreign key name symbol for key in join table that points to
         # current table's primary key (or :left_primary_key column).
         def default_left_key
@@ -245,6 +272,11 @@ module Sequel
           self[:left_primary_key]
         end
     
+        # many_to_many associations need to select a key in an associated table to eagerly load
+        def eager_loading_use_associated_key?
+          true
+        end
+
         # Whether the associated object needs a primary key to be added/removed,
         # true for many_to_many associations.
         def need_associated_primary_key?
@@ -258,8 +290,9 @@ module Sequel
           right_key = self[:right_key]
           join_table = self[:join_table]
           associated_class.all_association_reflections.each do |assoc_reflect|
-            if assoc_reflect[:type] == :many_to_many && assoc_reflect[:left_key] == right_key \
-               && assoc_reflect[:right_key] == left_key && assoc_reflect[:join_table] == join_table
+            if assoc_reflect[:type] == :many_to_many && assoc_reflect[:left_key] == right_key &&
+               assoc_reflect[:right_key] == left_key && assoc_reflect[:join_table] == join_table &&
+               assoc_reflect.associated_class == self[:model]
               return self[:reciprocal] = assoc_reflect[:name]
             end
           end
@@ -376,6 +409,9 @@ module Sequel
         #     before a new item is added to the association.
         #   - :before_remove - Symbol, Proc, or array of both/either specifying a callback to call
         #     before an item is removed from the association.
+        #   - :cartesian_product_number - he number of joins completed by this association that could cause more
+        #     than one row for each row in the current table (default: 0 for many_to_one associations,
+        #     1 for *_to_many associations).
         #   - :class - The associated class or its name. If not
         #     given, uses the association's name, which is camelized (and
         #     singularized unless the type is :many_to_one)
@@ -532,13 +568,20 @@ module Sequel
           association_reflections.keys
         end
 
-        # Modify and return eager loading dataset based on association options
+        # Modify and return eager loading dataset based on association options. Options:
         def eager_loading_dataset(opts, ds, select, associations)
           ds = ds.select(*select) if select
-          ds = ds.filter(opts[:conditions]) if opts[:conditions]
+          if c = opts[:conditions]
+            ds = (c.is_a?(Array) && !Sequel.condition_specifier?(c)) ? ds.filter(*c) : ds.filter(c)
+          end
           ds = ds.order(*opts[:order]) if opts[:order]
           ds = ds.eager(opts[:eager]) if opts[:eager]
-          ds = ds.eager_graph(opts[:eager_graph]) if opts[:eager_graph]
+          if opts[:eager_graph]
+            ds = ds.eager_graph(opts[:eager_graph])
+            ds = ds.add_graph_aliases(opts.associated_key_alias=>[opts.associated_class.table_name, opts.associated_key_alias, SQL::QualifiedIdentifier.new(opts.associated_key_table, opts.associated_key_column)]) if opts.eager_loading_use_associated_key?
+          else
+            ds.select_more(SQL::AliasedExpression.new(SQL::QualifiedIdentifier.new(opts.associated_key_table, opts.associated_key_column), opts.associated_key_alias)) if opts.eager_loading_use_associated_key?
+          end
           ds = ds.eager(associations) unless Array(associations).empty?
           ds = opts[:eager_block].call(ds) if opts[:eager_block]
           ds
@@ -604,9 +647,9 @@ module Sequel
           right = (opts[:right_key] ||= opts.default_right_key)
           left_pk = (opts[:left_primary_key] ||= self.primary_key)
           opts[:class_name] ||= camelize(singularize(name))
+          opts[:cartesian_product_number] ||= 1
           join_table = (opts[:join_table] ||= opts.default_join_table)
-          left_key_alias = opts[:left_key_alias] ||= :x_foreign_key_x
-          left_key_select = opts[:left_key_select] ||= SQL::QualifiedIdentifier.new(join_table, left).as(opts[:left_key_alias])
+          left_key_alias = opts[:left_key_alias] ||= opts.default_associated_key_alias
           graph_jt_conds = opts[:graph_join_table_conditions] = opts[:graph_join_table_conditions] ? opts[:graph_join_table_conditions].to_a : []
           opts[:graph_join_table_join_type] ||= opts[:graph_join_type]
           opts[:after_load].unshift(:array_uniq!) if opts[:uniq]
@@ -616,7 +659,7 @@ module Sequel
           opts[:eager_loader] ||= proc do |key_hash, records, associations|
             h = key_hash[left_pk]
             records.each{|object| object.associations[name] = []}
-            model.eager_loading_dataset(opts, opts.associated_class.inner_join(join_table, [[right, opts.right_primary_key], [left, h.keys]]), Array(opts.select) + Array(left_key_select), associations).all do |assoc_record|
+            model.eager_loading_dataset(opts, opts.associated_class.inner_join(join_table, [[right, opts.right_primary_key], [left, h.keys]]), Array(opts.select), associations).all do |assoc_record|
               next unless objects = h[assoc_record.values.delete(left_key_alias)]
               objects.each{|object| object.associations[name].push(assoc_record)}
             end
@@ -661,6 +704,7 @@ module Sequel
           model = self
           opts[:key] = opts.default_key unless opts.include?(:key)
           key = opts[:key]
+          opts[:cartesian_product_number] ||= 0
           opts[:class_name] ||= camelize(name)
           opts[:dataset] ||= proc do
             klass = opts.associated_class
@@ -730,6 +774,7 @@ module Sequel
           use_only_conditions = opts.include?(:graph_only_conditions)
           only_conditions = opts[:graph_only_conditions]
           conditions = opts[:graph_conditions]
+          opts[:cartesian_product_number] ||= 1
           graph_block = opts[:graph_block]
           opts[:eager_grapher] ||= proc do |ds, assoc_alias, table_alias|
             ds = ds.graph(opts.associated_class, use_only_conditions ? only_conditions : [[key, primary_key]] + conditions, :select=>select, :table_alias=>assoc_alias, :join_type=>join_type, :implicit_qualifier=>table_alias, &graph_block)
@@ -790,6 +835,13 @@ module Sequel
 
       # Private instance methods used to implement the associations support.
       module InstanceMethods
+        # Used internally by the associations code, like pk but doesn't raise
+        # an Error if the model has no primary key.
+        def pk_or_nil
+          key = primary_key
+          key.is_a?(Array) ? key.map{|k| @values[k]} : @values[key]
+        end
+
         private
 
         # Backbone behind association dataset methods
@@ -801,7 +853,9 @@ module Sequel
           ds.association_reflection = opts
           opts[:extend].each{|m| ds.extend(m)}
           ds = ds.select(*opts.select) if opts.select
-          ds = ds.filter(opts[:conditions]) if opts[:conditions]
+          if c = opts[:conditions]
+            ds = (c.is_a?(Array) && !Sequel.condition_specifier?(c)) ? ds.filter(*c) : ds.filter(c)
+          end
           ds = ds.order(*opts[:order]) if opts[:order]
           ds = ds.limit(*opts[:limit]) if opts[:limit]
           ds = ds.eager(*opts[:eager]) if opts[:eager]
@@ -810,6 +864,19 @@ module Sequel
           ds
         end
 
+        # Return the associated objects from the dataset, without callbacks, reciprocals, and caching.
+        def _load_associated_objects(opts)
+          if opts.returns_array?
+            send(opts.dataset_method).all
+          else
+            if !opts[:key]
+              send(opts.dataset_method).all.first
+            elsif send(opts[:key])
+              send(opts.dataset_method).first
+            end
+          end
+        end
+
         # Add the given associated object to the given association
         def add_associated_object(opts, o)
           raise(Sequel::Error, "model object #{model} does not have a primary key") unless pk
@@ -840,21 +907,13 @@ module Sequel
           a.uniq!
         end
 
-        # Load the associated objects using the dataset
+        # Load the associated objects using the dataset, handling callbacks, reciprocals, and caching.
         def load_associated_objects(opts, reload=false)
           name = opts[:name]
           if associations.include?(name) and !reload
             associations[name]
           else
-            objs = if opts.returns_array?
-              send(opts.dataset_method).all
-            else
-              if !opts[:key]
-                send(opts.dataset_method).all.first
-              elsif send(opts[:key])
-                send(opts.dataset_method).first
-              end
-            end
+            objs = _load_associated_objects(opts)
             run_association_callbacks(opts, :after_load, objs)
             objs.each{|o| add_reciprocal_object(opts, o)} if opts.set_reciprocal_to_self?
             associations[name] = objs
@@ -1040,7 +1099,7 @@ module Sequel
             # :requirements - array of requirements for this association
             # :alias_association_type_map - the type of association for this association
             # :alias_association_name_map - the name of the association for this association
-            clone(:eager_graph=>{:requirements=>{}, :master=>model.table_name, :alias_association_type_map=>{}, :alias_association_name_map=>{}, :reciprocals=>{}})
+            clone(:eager_graph=>{:requirements=>{}, :master=>model.table_name, :alias_association_type_map=>{}, :alias_association_name_map=>{}, :reciprocals=>{}, :cartesian_product_number=>0})
           end
           ds.eager_graph_associations(ds, model, table_name, [], *associations)
         end
@@ -1069,6 +1128,7 @@ module Sequel
           eager_graph[:requirements][assoc_table_alias] = requirements.dup
           eager_graph[:alias_association_name_map][assoc_table_alias] = assoc_name
           eager_graph[:alias_association_type_map][assoc_table_alias] = r.returns_array?
+          eager_graph[:cartesian_product_number] += r[:cartesian_product_number] || 2
           ds = ds.eager_graph_associations(ds, r.associated_class, assoc_table_alias, requirements + [assoc_table_alias], *associations) unless associations.empty?
           ds
         end
@@ -1143,7 +1203,7 @@ module Sequel
           records = []
           record_graphs.each do |record_graph|
             primary_record = record_graph[master]
-            key = primary_record.pk || primary_record.values.sort_by{|x| x[0].to_s}
+            key = primary_record.pk_or_nil || primary_record.values.sort_by{|x| x[0].to_s}
             if cached_pr = records_map[master][key]
               primary_record = cached_pr
             else
@@ -1156,7 +1216,7 @@ module Sequel
           end
       
           # Remove duplicate records from all associations if this graph could possibly be a cartesian product
-          eager_graph_make_associations_unique(records, dependency_map, alias_map, type_map) if type_map.values.select{|v| v}.length > 1
+          eager_graph_make_associations_unique(records, dependency_map, alias_map, type_map) if eager_graph[:cartesian_product_number] > 1
           
           # Replace the array of object graphs with an array of model objects
           record_graphs.replace(records)
@@ -1167,6 +1227,7 @@ module Sequel
         # N (starting at 0 and increasing until an unused one is found).
         def eager_unique_table_alias(ds, table_alias)
           used_aliases = ds.opts[:from]
+          used_aliases += ds.opts[:join].map{|j| j.table_alias || j.table} if ds.opts[:join]
           graph = ds.opts[:graph]
           used_aliases += graph[:table_aliases].keys if graph
           if used_aliases.include?(table_alias)
@@ -1200,21 +1261,20 @@ module Sequel
           end
           dependency_map.each do |ta, deps|
             next unless rec = record_graph[ta]
-            key = rec.pk || rec.values.sort_by{|x| x[0].to_s}
+            key = rec.pk_or_nil || rec.values.sort_by{|x| x[0].to_s}
             if cached_rec = records_map[ta][key]
               rec = cached_rec
             else
-              records_map[ta][rec.pk] = rec
+              records_map[ta][key] = rec
             end
             assoc_name = alias_map[ta]
-            case type_map[ta]
-            when false
-              current.associations[assoc_name] = rec
-            else
+            if type_map[ta]
               current.associations[assoc_name].push(rec) 
               if reciprocal = reciprocal_map[ta]
                 rec.associations[reciprocal] = current
               end
+            else
+              current.associations[assoc_name] = rec
             end
             # Recurse into dependencies of the current object
             eager_graph_build_associations_graph(deps, alias_map, type_map, reciprocal_map, records_map, rec, record_graph)
@@ -1229,12 +1289,11 @@ module Sequel
         def eager_graph_make_associations_unique(records, dependency_map, alias_map, type_map)
           records.each do |record|
             dependency_map.each do |ta, deps|
-              list = if !type_map[ta]
-                item = record.send(alias_map[ta])
-                [item] if item
-              else
-                list = record.send(alias_map[ta])
+              list = record.send(alias_map[ta])
+              list = if type_map[ta]
                 list.uniq!
+              else
+                [list] if list
               end
               # Recurse into dependencies
               eager_graph_make_associations_unique(list, deps, alias_map, type_map) if list
@@ -1260,10 +1319,8 @@ module Sequel
         end
       
         # Eagerly load all specified associations 
-        def eager_load(a)
+        def eager_load(a, eager_assoc=@opts[:eager])
           return if a.empty?
-          # All associations to eager load
-          eager_assoc = @opts[:eager]
           # Key is foreign/primary key name symbol
           # Value is hash with keys being foreign/primary key values (generally integers)
           #  and values being an array of current model objects with that
@@ -1293,6 +1350,7 @@ module Sequel
         def post_load(all_records)
           eager_graph_build_associations(all_records) if @opts[:eager_graph]
           eager_load(all_records) if @opts[:eager]
+          super
         end
       end
     end