sequel 5.38.0 → 5.43.0

data/doc/release_notes/5.43.0.txt

@@ -0,0 +1,98 @@
+= New Features
+
+* A column_encryption plugin has been added to support encrypting the
+  content of individual columns in a table.
+
+  Column values are encrypted with AES-256-GCM using a per-value
+  cipher key derived from a key provided in the configuration using
+  HMAC-SHA256.
+
+  If you would like to support encryption of columns in more than one
+  model, you should probably load the plugin into the parent class of
+  your models and specify the keys:
+ 
+    Sequel::Model.plugin :column_encryption do |enc|
+      enc.key 0, ENV["SEQUEL_COLUMN_ENCRYPTION_KEY"]
+    end
+ 
+  This specifies a single master encryption key.  Unless you are
+  actively rotating keys, it is best to use a single master key.
+ 
+  In the above call, 0 is the id of the key, and
+  ENV["SEQUEL_COLUMN_ENCRYPTION_KEY"] is the content of the key, which
+  must be a string with exactly 32 bytes. As indicated, this key
+  should not be hardcoded or otherwise committed to the source control
+  repository.
+ 
+  For models that need encrypted columns, you load the plugin again,
+  but specify the columns to encrypt:
+ 
+    ConfidentialModel.plugin :column_encryption do |enc|
+      enc.column :encrypted_column_name
+      enc.column :searchable_column_name, searchable: true
+      enc.column :ci_searchable_column_name, searchable: :case_insensitive
+    end
+ 
+  With this, all three specified columns (encrypted_column_name, 
+  searchable_column_name, and ci_searchable_column_name) will be
+  marked as encrypted columns.  When you run the following code:
+ 
+    ConfidentialModel.create(
+      encrypted_column_name: 'These',
+      searchable_column_name: 'will be',
+      ci_searchable_column_name: 'Encrypted'
+    )
+ 
+  It will save encrypted versions to the database.
+  encrypted_column_name will not be searchable, searchable_column_name
+  will be searchable with an exact match, and
+  ci_searchable_column_name will be searchable with a case insensitive
+  match.
+ 
+  To search searchable encrypted columns, use with_encrypted_value.
+  This example code will return the model instance created in the code
+  example in the previous section:
+ 
+    ConfidentialModel.
+      with_encrypted_value(:searchable_column_name, "will be")
+      with_encrypted_value(:ci_searchable_column_name, "encrypted").
+      first
+
+  To rotate encryption keys, add a new key above the existing key,
+  with a new key ID:
+ 
+    Sequel::Model.plugin :column_encryption do |enc|
+      enc.key 1, ENV["SEQUEL_COLUMN_ENCRYPTION_KEY"]
+      enc.key 0, ENV["SEQUEL_OLD_COLUMN_ENCRYPTION_KEY"]
+    end
+ 
+  Newly encrypted data will then use the new key.  Records encrypted
+  with the older key will still be decrypted correctly.
+ 
+  To force reencryption for existing records that are using the older
+  key, you can use the needing_reencryption dataset method and the
+  reencrypt instance method. For a small number of records, you can
+  probably do:
+ 
+    ConfidentialModel.needing_reencryption.all(&:reencrypt)
+ 
+  With more than a small number of records, you'll want to do this in
+  batches.  It's possible you could use an approach such as:
+ 
+    ds = ConfidentialModel.needing_reencryption.limit(100)
+    true until ds.all(&:reencrypt).empty?
+ 
+  After all values have been reencrypted for all models, and no models
+  use the older encryption key, you can remove it from the
+  configuration:
+ 
+    Sequel::Model.plugin :column_encryption do |enc|
+      enc.key 1, ENV["SEQUEL_COLUMN_ENCRYPTION_KEY"]
+    end
+
+  The column_encryption plugin supports encrypting serialized data,
+  as well as enforcing uniquenss of searchable encrypted columns
+  (in the absence of key rotation).  By design, it does not support
+  compression, mixing encrypted and unencrypted data in the same
+  column, or support arbitrary encryption ciphers.  See the plugin
+  documentation for more details.

data/doc/sql.rdoc

@@ -544,7 +544,7 @@ On some databases, you can specify null ordering:
 
 === All Columns (.*)
 
-To select all columns in a table, Sequel supports the * method on identifiers and qualified without an argument:
+To select all columns in a table, Sequel supports the * method on identifiers and qualified identifiers without an argument:
 
   Sequel[:table].*          # "table".*
   Sequel[:schema][:table].* # "schema"."table".*

data/doc/testing.rdoc

@@ -157,6 +157,8 @@ The SEQUEL_INTEGRATION_URL environment variable specifies the Database connectio
 
 === Other
 
+SEQUEL_ASYNC_THREAD_POOL :: Use the async_thread_pool extension when running the specs
+SEQUEL_ASYNC_THREAD_POOL_PREEMPT :: Use the async_thread_pool extension when running the specs, with the :preempt_async_thread option
 SEQUEL_COLUMNS_INTROSPECTION :: Use the columns_introspection extension when running the specs
 SEQUEL_CONNECTION_VALIDATOR :: Use the connection validator extension when running the specs
 SEQUEL_DUPLICATE_COLUMNS_HANDLER :: Use the duplicate columns handler extension with value given when running the specs

data/lib/sequel/adapters/ado.rb

@@ -195,10 +195,25 @@ module Sequel
         end
 
         @conversion_procs = CONVERSION_PROCS.dup
+        @conversion_procs[AdDBTimeStamp] = method(:adb_timestamp_to_application_timestamp)
 
         super
       end
 
+      def adb_timestamp_to_application_timestamp(v)
+        # This hard codes a timestamp_precision of 6 when converting.
+        # That is the default timestamp_precision, but the ado/mssql adapter uses a timestamp_precision
+        # of 3.  However, timestamps returned by ado/mssql have nsec values that end up rounding to a
+        # the same value as if a timestamp_precision of 3 was hard coded (either xxx999yzz, where y is
+        # 5-9 or xxx000yzz where y is 0-4).
+        #
+        # ADO subadapters should override this they would like a different timestamp precision and the
+        # this code does not work for them (for example, if they provide full nsec precision).
+        #
+        # Note that fractional second handling for WIN32OLE objects is not correct on ruby <2.2
+        to_application_timestamp([v.year, v.month, v.day, v.hour, v.min, v.sec, (v.nsec/1000.0).round * 1000])
+      end
+
       def dataset_class_default
         Dataset
       end
@@ -233,23 +248,8 @@ module Sequel
           cols = []
           conversion_procs = db.conversion_procs
 
-          ts_cp = nil
           recordset.Fields.each do |field|
-            type = field.Type
-            cp = if type == AdDBTimeStamp
-              ts_cp ||= begin
-                nsec_div = 1000000000.0/(10**(timestamp_precision))
-                nsec_mul = 10**(timestamp_precision+3)
-                meth = db.method(:to_application_timestamp)
-                lambda do |v|
-                  # Fractional second handling is not correct on ruby <2.2
-                  meth.call([v.year, v.month, v.day, v.hour, v.min, v.sec, (v.nsec/nsec_div).round * nsec_mul])
-                end
-              end
-            else
-              conversion_procs[type]
-            end
-            cols << [output_identifier(field.Name), cp]
+            cols << [output_identifier(field.Name), conversion_procs[field.Type]]
           end
 
           self.columns = cols.map(&:first)

data/lib/sequel/adapters/jdbc.rb

@@ -71,11 +71,11 @@ module Sequel
     class TypeConvertor
       CONVERTORS = convertors = {}
       %w'Boolean Float Double Int Long Short'.each do |meth|
-        x = convertors[meth.to_sym] = Object.new
+        x = x = convertors[meth.to_sym] = Object.new
         class_eval("def x.call(r, i) v = r.get#{meth}(i); v unless r.wasNull end", __FILE__, __LINE__)
       end
       %w'Object Array String Time Date Timestamp BigDecimal Blob Bytes Clob'.each do |meth|
-        x = convertors[meth.to_sym] = Object.new
+        x = x = convertors[meth.to_sym] = Object.new
         class_eval("def x.call(r, i) r.get#{meth}(i) end", __FILE__, __LINE__)
       end
       x = convertors[:RubyTime] = Object.new

data/lib/sequel/adapters/shared/mssql.rb

@@ -244,6 +244,16 @@ module Sequel
       
       private
       
+      # Add CLUSTERED or NONCLUSTERED as needed
+      def add_clustered_sql_fragment(sql, opts)
+        clustered = opts[:clustered]
+        unless clustered.nil?
+          sql += " #{'NON' unless clustered}CLUSTERED"
+        end
+
+        sql
+      end
+    
       # Add dropping of the default constraint to the list of SQL queries.
       # This is necessary before dropping the column or changing its type.
       def add_drop_default_constraint_sql(sqls, table, column)
@@ -284,7 +294,7 @@ module Sequel
         when :set_column_null
           sch = schema(table).find{|k,v| k.to_s == op[:name].to_s}.last
           type = sch[:db_type]
-          if [:string, :decimal].include?(sch[:type]) && !["text", "ntext"].include?(type) && (size = (sch[:max_chars] || sch[:column_size]))
+          if [:string, :decimal, :blob].include?(sch[:type]) && !["text", "ntext"].include?(type) && (size = (sch[:max_chars] || sch[:column_size]))
             size = "MAX" if size == -1
             type += "(#{size}#{", #{sch[:scale]}" if sch[:scale] && sch[:scale].to_i > 0})"
           end
@@ -396,6 +406,11 @@ module Sequel
         super.with_quote_identifiers(true)
       end
       
+      # Handle clustered and nonclustered primary keys
+      def primary_key_constraint_sql_fragment(opts)
+        add_clustered_sql_fragment(super, opts)
+      end
+      
       # Use sp_rename to rename the table
       def rename_table_sql(name, new_name)
         "sp_rename #{literal(quote_schema_table(name))}, #{quote_identifier(schema_and_table(new_name).pop)}"
@@ -492,6 +507,11 @@ module Sequel
         :'varbinary(max)'
       end
       
+      # Handle clustered and nonclustered unique constraints
+      def unique_constraint_sql_fragment(opts)
+        add_clustered_sql_fragment(super, opts)
+      end
+
       # MSSQL supports views with check option, but not local.
       def view_with_check_option_support
         true

data/lib/sequel/adapters/shared/postgres.rb

@@ -846,7 +846,7 @@ module Sequel
       # :schema :: The schema to search
       # :server :: The server to use
       def tables(opts=OPTS, &block)
-        pg_class_relname('r', opts, &block)
+        pg_class_relname(['r', 'p'], opts, &block)
       end
 
       # Check whether the given type name string/symbol (e.g. :hstore) is supported by
@@ -1500,9 +1500,11 @@ module Sequel
       # disallowed or there is a size specified, use the varchar type.
       # Otherwise use the text type.
       def type_literal_generic_string(column)
-        if column[:fixed]
+        if column[:text]
+          :text
+        elsif column[:fixed]
           "char(#{column[:size]||255})"
-        elsif column[:text] == false or column[:size]
+        elsif column[:text] == false || column[:size]
           "varchar(#{column[:size]||255})"
         else
           :text

data/lib/sequel/adapters/shared/sqlite.rb

@@ -561,7 +561,7 @@ module Sequel
       Dataset.def_sql_method(self, :delete, [['if db.sqlite_version >= 30803', %w'with delete from where'], ["else", %w'delete from where']])
       Dataset.def_sql_method(self, :insert, [['if db.sqlite_version >= 30803', %w'with insert conflict into columns values on_conflict'], ["else", %w'insert conflict into columns values']])
       Dataset.def_sql_method(self, :select, [['if opts[:values]', %w'with values compounds'], ['else', %w'with select distinct columns from join where group having window compounds order limit lock']])
-      Dataset.def_sql_method(self, :update, [['if db.sqlite_version >= 30803', %w'with update table set where'], ["else", %w'update table set where']])
+      Dataset.def_sql_method(self, :update, [['if db.sqlite_version >= 33300', %w'with update table set from where'], ['elsif db.sqlite_version >= 30803', %w'with update table set where'], ["else", %w'update table set where']])
 
       def cast_sql_append(sql, expr, type)
         if type == Time or type == DateTime
@@ -753,6 +753,11 @@ module Sequel
         false
       end
 
+      # SQLite does not support deleting from a joined dataset
+      def supports_deleting_joins?
+        false
+      end
+
       # SQLite does not support INTERSECT ALL or EXCEPT ALL
       def supports_intersect_except_all?
         false
@@ -763,6 +768,11 @@ module Sequel
         false
       end
       
+      # SQLite 3.33.0 supports modifying joined datasets
+      def supports_modifying_joins?
+        db.sqlite_version >= 33300
+      end
+
       # SQLite does not support multiple columns for the IN/NOT IN operators
       def supports_multiple_column_in?
         false
@@ -825,6 +835,13 @@ module Sequel
         end
       end
 
+      # Raise an InvalidOperation exception if insert is not allowed for this dataset.
+      def check_insert_allowed!
+        raise(InvalidOperation, "Grouped datasets cannot be modified") if opts[:group]
+        raise(InvalidOperation, "Joined datasets cannot be modified") if joined_dataset?
+      end
+      alias check_delete_allowed! check_insert_allowed!
+
       # SQLite supports a maximum of 500 rows in a VALUES clause.
       def default_import_slice
         500
@@ -944,6 +961,23 @@ module Sequel
       def _truncate_sql(table)
         "DELETE FROM #{table}"
       end
+
+      # Use FROM to specify additional tables in an update query
+      def update_from_sql(sql)
+        if(from = @opts[:from][1..-1]).empty?
+          raise(Error, 'Need multiple FROM tables if updating/deleting a dataset with JOINs') if @opts[:join]
+        else
+          sql << ' FROM '
+          source_list_append(sql, from)
+          select_join_sql(sql)
+        end
+      end
+
+      # Only include the primary table in the main update clause
+      def update_table_sql(sql)
+        sql << ' '
+        source_list_append(sql, @opts[:from][0..0])
+      end
     end
   end
 end

data/lib/sequel/database/misc.rb

@@ -213,8 +213,7 @@ module Sequel
       Sequel.extension(*exts)
       exts.each do |ext|
         if pr = Sequel.synchronize{EXTENSIONS[ext]}
-          unless Sequel.synchronize{@loaded_extensions.include?(ext)}
-            Sequel.synchronize{@loaded_extensions << ext}
+          if Sequel.synchronize{@loaded_extensions.include?(ext) ? false : (@loaded_extensions << ext)}
             pr.call(self)
           end
         else

data/lib/sequel/database/schema_generator.rb

@@ -143,8 +143,14 @@ module Sequel
       # :identity :: Create an identity column.
       #
       # MySQL specific options:
+      #
       # :generated_type :: Set the type of column when using :generated_always_as,
       #                    should be :virtual or :stored to force a type.
+      #
+      # Microsoft SQL Server specific options:
+      #
+      # :clustered :: When using :primary_key or :unique, marks the primary key or unique
+      #               constraint as CLUSTERED (if true), or NONCLUSTERED (if false).
       def column(name, type, opts = OPTS)
         columns << {:name => name, :type => type}.merge!(opts)
         if index_opts = opts[:index]
@@ -153,7 +159,7 @@ module Sequel
         nil
       end
       
-      # Adds a named constraint (or unnamed if name is nil),
+      # Adds a named CHECK constraint (or unnamed if name is nil),
       # with the given block or args. To provide options for the constraint, pass
       # a hash as the first argument.
       #
@@ -161,6 +167,15 @@ module Sequel
       #   # CONSTRAINT blah CHECK num >= 1 AND num <= 5
       #   constraint({name: :blah, deferrable: true}, num: 1..5)
       #   # CONSTRAINT blah CHECK num >= 1 AND num <= 5 DEFERRABLE INITIALLY DEFERRED
+      #
+      # If the first argument is a hash, the following options are supported:
+      #
+      # Options:
+      # :name :: The name of the CHECK constraint
+      # :deferrable :: Whether the CHECK constraint should be marked DEFERRABLE.
+      #
+      # PostgreSQL specific options:
+      # :not_valid :: Whether the CHECK constraint should be marked NOT VALID.
       def constraint(name, *args, &block)
         opts = name.is_a?(Hash) ? name : {:name=>name}
         constraints << opts.merge(:type=>:check, :check=>block || args)

data/lib/sequel/database/schema_methods.rb

@@ -262,6 +262,10 @@ module Sequel
     #   # SELECT * FROM items WHERE foo
     #   # WITH CHECK OPTION
     #
+    #   DB.create_view(:bar_items, DB[:items].select(:foo), columns: [:bar])
+    #   # CREATE VIEW bar_items (bar) AS
+    #   # SELECT foo FROM items
+    #
     # Options:
     # :columns :: The column names to use for the view.  If not given,
     #             automatically determined based on the input dataset.
@@ -580,14 +584,14 @@ module Sequel
         sql << ' NULL'
       end
     end
-    
+
     # Add primary key SQL fragment to column creation SQL.
     def column_definition_primary_key_sql(sql, column)
       if column[:primary_key]
         if name = column[:primary_key_constraint_name]
           sql << " CONSTRAINT #{quote_identifier(name)}"
         end
-        sql << ' PRIMARY KEY'
+        sql << " " << primary_key_constraint_sql_fragment(column)
         constraint_deferrable_sql_append(sql, column[:primary_key_deferrable])
       end
     end
@@ -608,7 +612,7 @@ module Sequel
         if name = column[:unique_constraint_name]
           sql << " CONSTRAINT #{quote_identifier(name)}"
         end
-        sql << ' UNIQUE'
+        sql << ' ' << unique_constraint_sql_fragment(column)
         constraint_deferrable_sql_append(sql, column[:unique_deferrable])
       end
     end
@@ -656,11 +660,11 @@ module Sequel
         check = "(#{check})" unless check[0..0] == '(' && check[-1..-1] == ')'
         sql << "CHECK #{check}"
       when :primary_key
-        sql << "PRIMARY KEY #{literal(constraint[:columns])}"
+        sql << "#{primary_key_constraint_sql_fragment(constraint)} #{literal(constraint[:columns])}"
       when :foreign_key
         sql << column_references_table_constraint_sql(constraint.merge(:deferrable=>nil))
       when :unique
-        sql << "UNIQUE #{literal(constraint[:columns])}"
+        sql << "#{unique_constraint_sql_fragment(constraint)} #{literal(constraint[:columns])}"
       else
         raise Error, "Invalid constraint type #{constraint[:type]}, should be :check, :primary_key, :foreign_key, or :unique"
       end
@@ -892,6 +896,11 @@ module Sequel
       on_delete_clause(action)
     end
     
+    # Add fragment for primary key specification, separated for easier overridding.
+    def primary_key_constraint_sql_fragment(_)
+      'PRIMARY KEY'
+    end
+    
     # Proxy the quote_schema_table method to the dataset
     def quote_schema_table(table)
       schema_utility_dataset.quote_schema_table(table)
@@ -1047,6 +1056,11 @@ module Sequel
       "#{type}#{literal(Array(elements)) if elements}#{' UNSIGNED' if column[:unsigned]}"
     end
 
+    # Add fragment for unique specification, separated for easier overridding.
+    def unique_constraint_sql_fragment(_)
+      'UNIQUE'
+    end
+    
     # Whether clob should be used for String text: true columns.
     def uses_clob_for_text?
       false