sequel 4.8.0 → 4.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c96550377e172f2fe1bbca889e66dbf6f4ba2ea
-  data.tar.gz: 09b5f64bdd1e48a538caf87c359f0196c5cecf87
+  metadata.gz: 54a1904977f9d5f4de5e523dbd0ba244e8ef6203
+  data.tar.gz: 404f40b96e9fc774587485c727186aa6ab2c54b7
 SHA512:
-  metadata.gz: d089308d0d6ae3643cd8d84d6a67ad2f130dbafc8bafc1d0432c6ce0e0f496f50c76fed0db3a503a82afc889be9ba7c5639f971b763d1a0b9def6867939ac743
-  data.tar.gz: 11bcfe87e96fbd8c61b336994cd142ceec0d0fe5964ab94ea4d2886e7304843180377c488515d2f87dcaf9839974d988c9805c3872baecc5e7f9c373ea2d70fc
+  metadata.gz: 323f3bb4faea4a371565fecf027aa190a8b4b8232b2da1a0205fcdbccaac20decddbed540a992a4a7c19df78904483b4da3e70be408f3c432c4288357c5b37d3
+  data.tar.gz: cef071ae2cac67201d942e674905dfb266cf9054bf80f3d1aed43ce9e2e032a71325101406ce689592c757335a8f2471ce14f8102f4e178b37e6dd9777c12b40

data/CHANGELOG

@@ -1,3 +1,51 @@
+=== 4.9.0 (2014-04-01)
+
+* Recognize CHECK constraint violations on newer versions of SQLite (jeremyevans)
+
+* Do not attempt to eager load when calling Dataset#columns in the eager_each plugin (jeremyevans)
+
+* Support :driver option for jdbc adapter, for specifying driver class for cases where getConnection doesn't work (jeremyevans) (#785)
+
+* Massive speedup for PostgreSQL array parser (jeremyevans) (#788)
+
+* Add current_datetime_timestamp extension, for current Time/DateTime instances that are literalized as CURRENT_TIMESTAMP (jeremyevans)
+
+* Recognize additional unique constraint violations on SQLite (jeremyevans) (#782)
+
+* Don't remove column value when validating nested attributes for one_to_* association where association foreign key is the model's primary key (jeremyevans)
+
+* Add Dataset#disable_insert_returning on PostgreSQL for skipping implicit use of RETURNING (jeremyevans)
+
+* Automatically optimize Model.[], .with_pk, and .with_pk! for models with composite keys (jeremyevans)
+
+* Automatically optimize Model.[] when called with a hash (jeremyevans)
+
+* Automatically optimize Model.find, .first, and .first! when called with a single argument (jeremyevans)
+
+* Add Model.finder for creating optimized finder methods using Dataset::PlaceholderLiteralizer (jeremyevans)
+
+* Add Dataset::PlaceholderLiteralizer optimization framework (jeremyevans)
+
+* Add Dataset#with_sql_{each,all,first,single_value,insert,update} optimized methods (jeremyevans)
+
+* Make pg_array extension use correct type when typecasting column values for smallint, oid, real, character, and varchar arrays (jeremyevans)
+
+* Make Database#column_schema_to_ruby_default a public method in the schema_dumper extension (jeremyevans) (#776)
+
+* Fix multiple corner cases in the eager_graph support (jeremyevans) (#771)
+
+* Use streaming to implement paging for Dataset#paged_each in the mysql2 adapter (jeremyevans)
+
+* Use a cursor to implement paging for Dataset#paged_each in the postgres adapter (jeremyevans)
+
+* Add Database#create_join_table? and #create_join_table! for consistency (jeremyevans)
+
+* Add Dataset#where_current_of to the postgres adapter for supporting updating rows based on a cursor's current position (jeremyevans)
+
+* Add Dataset#use_cursor :hold option in the postgres adapter for supporting cursor use outside of a transaction (jeremyevans)
+
+* Add Dataset#paged_each :strategy=>:filter option for increased performance (jeremyevans)
+
 === 4.8.0 (2014-03-01)
 
 * Add SQL::AliasedExpression#alias alias for #aliaz (jeremyevans)

data/doc/association_basics.rdoc

@@ -133,7 +133,7 @@ single join table.
     # Uses plural form of associated model name
     many_to_many :albums
 
-    # Uses plural form of associated model name
+    # Uses singular form of associated model name
     one_through_one :album
   end
 

data/doc/opening_databases.rdoc

@@ -283,6 +283,10 @@ The following additional options are supported:
                   Setting to false roughly doubles performance when selecting large numbers of rows.
                   Note that you can't provide this option inside the connection string (as that is passed
                   directly to JDBC), you have to pass it as a separate option.
+:driver :: Specify the Java driver class to use to connect to the database.  This only has
+           an effect if the database type is not recognized from the connection string,
+           and only helps cases where <tt>java.sql.DriverManager.getConnection</tt> does not
+           return a connection.
 :login_timeout :: Set the login timeout on the JDBC connection (in seconds).
 
 === mysql 

data/doc/postgresql.rdoc

@@ -44,14 +44,15 @@ pg_range :: ranges (for any scalar type), as a ruby Range-like object
 pg_row :: row-valued/composite types, as a ruby Hash-like or Sequel::Model object
 
 In general, these extensions just add support for Database objects to return retrieved
-column values as the appropriate type (<tt>postgres only</tt>), and support for literalizing
-the objects correctly for use in an SQL string, or using them as bound variable values (<tt>postgres/pg only</tt>).
+column values as the appropriate type (<tt>postgres and jdbc/postgres only</tt>), and support for literalizing
+the objects correctly for use in an SQL string, or using them as bound variable values (<tt>postgres/pg and jdbc/postgres only</tt>).
 
 There are also type-specific extensions that make it easy to use database functions
 and operators related to the type.  These extensions are:
 
 pg_array_ops :: array-related functions and operators
 pg_hstore_ops :: hstore-related functions and operators
+pg_json_ops :: json-related functions and operators
 pg_range_ops :: range-related functions and operators
 pg_row_ops :: row-valued/composite type syntax support
 
@@ -75,7 +76,7 @@ You can also add exclusion constraints in +alter_table+ blocks using add_exclusi
   end
   # ALTER TABLE "table" ADD CONSTRAINT "table_during_excl" EXCLUDE USING gist ("during" WITH &&)
 
-=== Adding Foreign Key Constraints Without Initial Validation
+=== Adding Foreign Key and Check Constraints Without Initial Validation
 
 You can add a <tt>:not_valid=>true</tt> option when adding constraints to existing tables so
 that it doesn't check if all current rows are valid:
@@ -83,16 +84,21 @@ that it doesn't check if all current rows are valid:
   DB.alter_table(:table) do
     # Assumes t_id column already exists
     add_foreign_key([:t_id], :table, :not_valid=>true, :name=>:table_fk)
+
+    constraint({:name=>:col_123, :not_valid=>true}, :col=>[1,2,3])
   end
   # ALTER TABLE "table" ADD CONSTRAINT "table_fk" FOREIGN KEY ("t_id") REFERENCES "table" NOT VALID
+  # ALTER TABLE "table" ADD CONSTRAINT "col_123" CHECK (col IN (1, 2, 3)) NOT VALID
 
 Such constraints will be enforced for newly inserted and updated rows, but not for existing rows. After
 all existing rows have been fixed, you can validate the constraint:
 
   DB.alter_table(:table) do
     validate_constraint(:table_fk)
+    validate_constraint(:col_123)
   end
   # ALTER TABLE "table" VALIDATE CONSTRAINT "table_fk"
+  # ALTER TABLE "table" VALIDATE CONSTRAINT "col_123"
 
 === Creating Indexes Concurrently
 
@@ -202,6 +208,17 @@ without keeping all rows in memory:
   # CLOSE sequel_cursor
   # COMMIT
 
+This support is used by default when using <tt>Dataset#paged_each</tt>.
+
+Using cursors, it is possible to update individual rows of a large dataset
+easily using the <tt>:rows_per_fetch=>1</tt> option in conjunction with
+<tt>Dataset#where_current_of</tt>.  This is useful if the logic needed to
+update the rows exists in the application and not in the database:
+
+  ds.use_cursor(:rows_per_fetch=>1).each do |row|
+    ds.where_current_of.update(:col=>new_col_value(row))
+  end
+
 === Truncate Modifiers
 
 Sequel supports PostgreSQL-specific truncate options:
@@ -261,6 +278,10 @@ notifications:
 
   DB.listen(:channel, :loop=>true){|channel| p channel}
 
+The +pg_static_cache_updater+ extension uses this support to automatically update
+the caches for models using the +static_cache+ plugin.  Look at the documentation of that
+plugin for details.
+
 === Locking Tables
 
 Sequel makes it easy to lock tables, though it is generally better to let the database
@@ -300,3 +321,6 @@ Then you can stream individual datasets:
 Or stream all datasets by default:
 
   DB.stream_all_queries = true
+
+When streaming is enabled, <tt>Dataset#paged_each</tt> will use streaming to implement
+paging.

data/doc/release_notes/4.9.0.txt

@@ -0,0 +1,190 @@
+= Performance Enhancements
+
+* Dataset::PlaceholderLiteralizer has been added as an optimization
+  framework.  This allows you to record changes to a given dataset
+  using placeholder arguments, and later quickly execute the query
+  providing values for the placeholders.  This is similar in idea
+  to prepared statements, except that the SQL for each query can
+  change depending on the values for the placeholders.
+
+  Using this optimization framework, generating the SQL for query
+  is about 3x faster, and since SQL generation time is a significant
+  portion of total time for simple queries, simple queries can
+  execute up to 50% faster.
+
+  There are two APIs for this optimization framework.  There is a
+  lower level dataset API:
+
+    loader = Sequel::Dataset::PlaceholderLiteralizer.
+     loader(DB[:items]) do |pl, ds|
+      ds.where(:id=>pl.arg).exclude(:name=>pl.arg).limit(1)
+    end
+
+    loader.first(1, "foo")
+    # SELECT * FROM items WHERE ((id = 1) AND (name != 'foo')) LIMIT 1 
+
+    loader.first([1, 2], %w"foo bar")
+    # SELECT * FROM items WHERE ((id IN (1, 2)) AND
+    #   (name NOT IN ('foo', 'bar'))) LIMIT 1
+
+  There is also a higher level model API (Model.finder):
+
+    class Item < Sequel::Model
+      # Given class method that returns a dataset
+      def self.by_id_and_not_name(id, not_name)
+        where(:id=>id).exclude(:name=>not_name)
+      end
+
+      # Create optimized method that returns first value
+      finder :by_id_and_not_name
+    end
+    
+    # Call optimized method
+    Album.first_by_id_and_not_name(1, 'foo')
+    # SELECT * FROM items WHERE ((id = 1) AND (name != 'foo')) LIMIT 1 
+
+  Model.finder defaults to creating a method that returns the first
+  matching row, but using the :type option you can create methods
+  that call each, all, or get.  There is also an option to choose the
+  method name (:name), as well as one to specify the number of
+  arguments to use if the method doesn't take a fixed number
+  (:arity).
+
+  Finally, Model.find, .first, and .first! now automatically use an
+  optimized finder if given a single argument. Model.[] uses an
+  optimized finder if given a single hash, and Model.[], .with_pk,
+  and .with_pk! use an optimized finder if the model has a composite
+  primary key.  In all of these cases, these methods are about 50%
+  faster than before.
+
+* The pure-ruby PostgreSQL array parser that ships with Sequel has
+  been replaced with a strscan-based parser.  This parser avoids
+  O(n^2) performance for arrays with multibyte strings, and in general
+  is much faster.  Parsing an array with a single string with 100,000
+  multibyte characters is about 1000x faster, and now about half the
+  speed of the C implementation in sequel_pg.
+
+* Dataset#paged_each now has a :strategy=>:filter option that
+  dramatically improves performance, especially if the columns
+  being ordered by are indexed.
+
+  Unfortunately, there are enough corner cases to this approach
+  that it cannot be used by default.  At the least, the dataset
+  needs to be selecting the columns it is ordering by, not aliasing
+  the columns it is ordering by in the SELECT clause, not have
+  NULLs in any of the columns being ordered by, and not itself use
+  a limit or offset.
+
+  If you are ordering by expressions that are not simple column
+  values, you can provide a :filter_value option proc that takes the
+  last retrieved row and array of order by expressions, and returns 
+  an array of values in the last retrieved row for those order by
+  expressions.
+
+* In the postgres adapter, Dataset#paged_each now automatically uses
+  a cursor for improved performance.
+
+* In the mysql2 adapter, Dataset#paged_each now automatically uses
+  streaming for improved performance, if streaming is supported.
+
+* Dataset#with_sql_{each,all,first,single_value,insert,update}
+  have been added.  These methods take specific SQL and execute
+  it on the database, returning the appropriate value.  They
+  are significantly faster than the previous approach of
+  with_sql(SQL).{each,all,first,single_value,insert,update},
+  as they don't require cloning the dataset.
+
+= New Features
+
+* Database#create_join_table! and #create_join_table? have been added,
+  for consistency with #create_table! and #create_table?.
+
+* A :hold option has been added to Dataset#use_cursor in the postgres
+  adapter, which uses WITH HOLD in the query, allowing for usage of
+  the cursor outside the enclosing transaction.  When :hold is used,
+  Sequel does not automatically use a transaction around the cursor
+  call.
+
+* Dataset#where_current_of has been added to the postgres adapter,
+  for updating rows based on a cursor's current position.  This can
+  be used to update a large dataset where new values depend on
+  some ruby method, without keeping all rows in memory.
+
+    ds = DB[:huge_table]
+    ds.use_cursor(:rows_per_fetch=>1).each do |row|
+      ds.where_current_of.update(:column=>ruby_method(row))
+    end
+
+* A current_datetime_timestamp extension has been added, for
+  creating Time/DateTime instances that are literalized as
+  CURRENT_TIMESTAMP.  When the dataset uses this extension, models
+  that use the touch and timestamps plugins will use
+  CURRENT_TIMESTAMP for the timestamps.
+
+* The jdbc adapter now supports a :driver option, useful when
+  Sequel doesn't have direct support for the underlying driver, and
+  where java.sql.DriverManager.getConnection does not work
+  correctly due to Java class loading issues.
+
+= Other Improvements
+
+* Multiple corner cases in Dataset#eager_graph have been fixed.
+
+* Calling Dataset#columns when using the eager_each plugin no
+  longer triggers eager loading.
+
+* Database#column_schema_to_ruby_default is now a public method
+  in the schema_dumper extension.
+
+* When validating associated objects for one_to_many and one_to_one
+  associations in the nested_attributes plugin, don't remove column
+  values if the association's foreign key is the associated model's
+  primary key.
+
+* On PostgreSQL, Dataset#disable_insert_returning has been added
+  back.  This disables the automatic use of RETURNING for INSERTs
+  for the dataset.  This is necessary in cases where INSERT
+  RETURNING doesn't work, such as PostgreSQL <8.2 (or PostgreSQL
+  variants that forked before 8.2), or when using partitioning
+  with trigger functions, or conditional rules.
+
+  Note that if you use disable_insert_returning, insert will not
+  return the autoincremented primary key.  You need to call
+  currval or lastval manually using the same connection to get
+  the value, or use nextval to get the value to use before
+  inserting.
+
+* The pg_array extension now uses the correct database type when
+  typecasting values for smallint, oid, real, character, and varchar
+  arrays.  Previously, Sequel did not use the correct database type
+  in some cases (e.g. text[] for a varchar[]), which resulted in
+  errors if the value was used in a filter expression.
+
+* Additional unique constraint violations are now recognized on
+  SQLite.
+
+* Check constraint violations are now recognized on SQLite >=3.8.2.
+
+* Adapters that emulate bitwise operators now do so using an append
+  only design, similar to how all other queries are built in Sequel.
+
+= Backwards Compatibility
+
+* In some cases Sequel no longer adds superfluous parentheses when
+  constructing SQL strings.  If you are testing for specific SQL,
+  this can cause test failures.
+
+* The pg_array extension no longer recognizes the :typecast_method
+  option when registering an array type.  The option allowed reuse
+  of an existing typecast method, but as that results in an incorrect
+  type at the database level, the option was fundementally broken.
+
+* The internals of the PostgreSQL array parser have changed.  If you
+  were relying on them, you'll need to update your code.
+
+* Dataset#complex_expression_arg_pairs private method now returns
+  nested expression objects instead of an already literalized string
+  in some cases.  Custom adapters that call this method will probably
+  need to be changed.  It's recommended that such adapters switch to
+  using the new Dataset#complex_expression_emulate_append method if
+  possible.

data/doc/security.rdoc

@@ -126,7 +126,7 @@ Note that for that type of query, Sequel generally encourages the following form
   DB[:table].where{|o| o.name > params[:id].to_s} # Safe
 
 Sequel's DSL supports a wide variety of SQL concepts, so it's possible to
-code most applications without every using raw SQL.
+code most applications without ever using raw SQL.
 
 A large number of dataset methods ultimately pass down their arguments to a filter
 method, even some you may not expect, so you should be careful.  At least the

data/doc/testing.rdoc

@@ -29,7 +29,7 @@ These run each test in its own transaction, the recommended way to test.
     end
   end
 
-=== RSpec 2, >=2.8
+=== RSpec >=2.8
 
   # Global around filters should work
   RSpec.configure do |c|
@@ -126,7 +126,7 @@ The order in which you delete/truncate the tables is important if you are using
 
 = Testing Sequel Itself
 
-Sequel has multiple separate test suites.  All test suites run under either RSpec 1 or RSpec 2.
+Sequel has multiple separate test suites.  All test suites run under rspec >=1.3.
 
 == rake spec
 

data/doc/validations.rdoc

@@ -510,6 +510,14 @@ Here, you don't care about validating the release date if there were validation
   album.errors.full_messages
   # => ["name cannot be empty"]
   
+Note that the column names used in the errors are used verbatim in the error messages.  If you want full control over the error messages, you can use +add+ with a literal string:
+
+  errors.add(:name, Sequel.lit("Album name is not valid"))
+  errors.full_messages
+  # => ["Album name is not valid"]
+
+Alternatively, feel free to override Sequel::Model::Errors#full_messages.  As long as it returns an array of strings, overridding it is completely safe.
+
 === +count+
 
 +count+ returns the total number of error messages in the errors.

data/lib/sequel/adapters/jdbc.rb

@@ -188,7 +188,7 @@ module Sequel
       # The type of database we are connecting to
       attr_reader :database_type
       
-      # The Java database driver we are using
+      # The Java database driver we are using (should be a Java class)
       attr_reader :driver
       
       # Whether to convert some Java types to ruby types when retrieving rows.
@@ -379,8 +379,10 @@ module Sequel
         
         resolved_uri = jndi? ? get_uri_from_jndi : uri
 
-        if match = /\Ajdbc:([^:]+)/.match(resolved_uri) and prok = DATABASE_SETUP[match[1].to_sym]
-          @driver = prok.call(self)
+        @driver = if (match = /\Ajdbc:([^:]+)/.match(resolved_uri)) && (prok = DATABASE_SETUP[match[1].to_sym])
+          prok.call(self)
+        else
+          @opts[:driver]
         end        
       end
       

data/lib/sequel/adapters/jdbc/derby.rb

@@ -173,8 +173,6 @@ module Sequel
         PAREN_OPEN = Dataset::PAREN_OPEN
         OFFSET = Dataset::OFFSET
         CAST_STRING_OPEN = "RTRIM(".freeze
-        BITCOMP_OPEN = "((0 - ".freeze
-        BITCOMP_CLOSE = ") - 1)".freeze
         BLOB_OPEN = "CAST(X'".freeze
         BLOB_CLOSE = "' AS BLOB)".freeze
         HSTAR = "H*".freeze
@@ -212,14 +210,10 @@ module Sequel
 
         def complex_expression_sql_append(sql, op, args)
           case op
-          when :%
-            sql << complex_expression_arg_pairs(args){|a, b| "MOD(#{literal(a)}, #{literal(b)})"}
+          when :%, :'B~'
+            complex_expression_emulate_append(sql, op, args)
           when :&, :|, :^, :<<, :>>
             raise Error, "Derby doesn't support the #{op} operator"
-          when :'B~'
-            sql << BITCOMP_OPEN
-            literal_append(sql, args.at(0))
-            sql << BITCOMP_CLOSE
           when :extract
             sql << args.at(0).to_s << PAREN_OPEN
             literal_append(sql, args.at(1))