sequel 2.3.0 → 2.4.0

data/CHANGELOG

@@ -1,3 +1,19 @@
+=== 2.4.0 (2008-08-06)
+
+* Handle Java::JavaSql::Date type in the JDBC adapter (jeremyevans)
+
+* Add support for read-only slave/writable master databases and database sharding (jeremyevans)
+
+* Remove InvalidExpression, InvalidFilter, InvalidJoinType, and WorkerStop exceptions (jeremyevans)
+
+* Add prepared statement/bound variable support (jeremyevans)
+
+* Fix anonymous column names in the ADO adapter (nusco)
+
+* Remove odbc_mssql adapter, use :db_type=>'mssql' option instead (jeremyevans)
+
+* Split MSSQL specific syntax into separate file, usable by ADO and ODBC adapters (nusco, jeremyevans)
+
 === 2.3.0 (2008-07-25)
 
 * Enable almost full support for MySQL using JDBC (jeremyevans)

data/README

@@ -6,6 +6,8 @@ Sequel is a lightweight database access toolkit for Ruby.
   for constructing database queries and table schemas.
 * Sequel also includes a lightweight but comprehensive ORM layer for
   mapping records to Ruby objects and handling associated records.
+* Sequel supports advanced database features such as prepared statements,
+  bound variables, master/slave configurations, and database sharding.
 * Sequel makes it easy to deal with multiple records without having
   to break your teeth on SQL.
 * Sequel currently has adapters for ADO, DB2, DBI, Informix, JDBC,
@@ -13,10 +15,11 @@ Sequel is a lightweight database access toolkit for Ruby.
 
 == Resources
 
+* {Website}[http://sequel.rubyforge.org]
 * {Source code}[http://github.com/jeremyevans/sequel]
 * {Bug tracking}[http://code.google.com/p/ruby-sequel/issues/list]
 * {Google group}[http://groups.google.com/group/sequel-talk]
-* {RDoc}[http://sequel.rubyforge.org]
+* {RDoc}[http://sequel.rubyforge.org/rdoc]
 
 To check out the source code:
   

data/Rakefile

@@ -8,14 +8,13 @@ require "spec/rake/spectask"
 include FileUtils
 
 NAME = 'sequel'
-VERS = '2.3.0'
-CLEAN.include ["**/.*.sw?", "pkg", ".config", "rdoc", "coverage"]
+VERS = '2.4.0'
+CLEAN.include ["**/.*.sw?", "pkg", ".config", "rdoc", "coverage", "www/public/*.html"]
 RDOC_OPTS = ["--quiet", "--line-numbers", "--inline-source", '--title', \
   'Sequel: The Database Toolkit for Ruby', '--main', 'README']
 
-##############################################################################
-# gem packaging and release
-##############################################################################
+# Gem Packaging and Release
+
 desc "Packages sequel"
 task :package=>[:clean]
 spec = Gem::Specification.new do |s|
@@ -64,25 +63,26 @@ task :release=>[:package] do
   sh %{rubyforge add_file sequel #{NAME} #{VERS} pkg/#{NAME}-#{VERS}.gem} 
 end
 
-##############################################################################
-# rdoc
-##############################################################################
+### RDoc
+
 Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_dir = "rdoc"
   rdoc.options += RDOC_OPTS
   rdoc.rdoc_files.add %w"README CHANGELOG COPYING lib/**/*.rb doc/*.rdoc"
 end
 
-desc "Update docs and upload to rubyforge.org"
-task :doc_rforge => [:rdoc]
-task :doc_rforge do
-  sh %{chmod -R g+w rdoc/*}
-  sh %{scp -rp rdoc/* rubyforge.org:/var/www/gforge-projects/sequel}
+### Website
+
+desc "Update sequel.rubyforge.org"
+task :website => [:rdoc]
+task :website do
+  sh %{www/make_www.rb}
+  sh %{scp -r www/public/* rubyforge.org:/var/www/gforge-projects/sequel/}
+  sh %{scp -r rdoc/* rubyforge.org:/var/www/gforge-projects/sequel/rdoc/}
 end
 
-##############################################################################
-# specs
-##############################################################################
+### Specs
+
 lib_dir = File.join(File.dirname(__FILE__), 'lib')
 fixRUBYLIB = Proc.new{ENV['RUBYLIB'] ? (ENV['RUBYLIB'] += ":#{lib_dir}") : (ENV['RUBYLIB'] = lib_dir)}
 sequel_core_specs = "spec/sequel_core/*_spec.rb"
@@ -141,9 +141,7 @@ task :dcov do
   sh %{find lib -name '*.rb' | xargs dcov}
 end
 
-##############################################################################
-# Statistics
-##############################################################################
+### Statistics
 
 STATS_DIRECTORIES = [
   %w(Code   lib/),

data/doc/prepared_statements.rdoc

@@ -0,0 +1,104 @@
+= Prepared Statements and Bound Variables
+
+Starting with version 2.4.0, Sequel has support for prepared statements and
+bound variables.  No matter which database you are using, the Sequel prepared
+statement/bound variable API remains exactly the same.  There is native support
+for prepared statements/bound variables on the following databases:
+
+* PostgreSQL (using the pg driver, requires type specifiers)
+* MySQL (prepared statements only, as the ruby mysql driver doesn't support
+  bound variables)
+* SQLite (a new native prepared statement is used for each call, though)
+* JDBC (using the postgresql, mysql, or sqlite databases, and possibly others)
+
+Support on other databases is emulated via the usual string interpolation.
+
+== Placeholders
+
+Generally, when using prepared statements (and certainly when using bound
+variables), you need to put placeholders in your SQL to indicate where you
+want your bound arguments to appear.  Database support and syntax vary
+significantly for placeholders (e.g. :name, $1, ?).  Sequel abstracts all of
+that and allows you to specify placeholders by using the :$name format for
+placeholders, e.g.:
+
+  ds = DB[:items].filter(:name=>:$name)
+
+== Bound Variables
+
+Using bound variables for this query is simple:
+
+  ds.call(:select, :name=>'Jim')
+
+This will do the equivalent of selecting records that have the name 'Jim'. It
+returns all records, and can take a block that is passed to Dataset#all.
+
+Deleting or returning the first record works similarly:
+
+  ds.call(:first, :name=>'Jim') # First record with name 'Jim'
+  ds.call(:delete, :name=>'Jim') # Delete records with name 'Jim'
+
+For inserting/updating records, you should also specify a value hash, which
+may itself contain placeholders:
+
+  # Insert record with 'Jim', note that the previous filter is ignored
+  ds.call(:insert, {:name=>'Jim'}, :name=>:$name)
+  # Change name to 'Bob' for all records with name of 'Jim'
+  ds.call(:update, {:name=>'Jim', :new_name=>'Bob'}, :name=>$:new_name)
+
+== Prepared Statements
+
+Prepared statement support is similar to bound variable support, but you
+use Dataset#prepare with a name, and Dataset#call later with the values:
+
+  ds = DB[:items].filter(:name=>:$name)
+  ps = ds.prepare(:select, :select_by_name)
+  ps.call(:name=>'Jim')
+  DB.call(:select_by_name, :name=>'Jim') # same as above
+
+The Dataset#prepare method returns a prepared statement, and also stores a
+copy of the prepared statement in the database for later use.  For insert
+and update queries, the hash to insert/update is passed to prepare:
+
+  ps1 = DB[:items].prepare(:insert, :insert_with_name, :name=>:$name)
+  ps1.call(:name=>'Jim')
+  DB.call(:insert_with_name, :name=>'Jim') # same as above
+  ds = DB[:items].filter(:name=>:$name)
+  ps2 = ds.prepare(:update, :update_name, :name=>:$new_name)
+  ps2.call(:name=>'Jim', :new_name=>'Bob')
+  DB.call(:update_name, :name=>'Jim', :new_name=>'Bob') # same as above
+
+== Database support
+
+=== PostgreSQL
+
+If you are using the ruby-postgres or postgres-pr driver, PostgreSQL uses the
+default emulated support.  If you are using ruby-pg, there is native support,
+but it requires type specifiers most of the time.  This is easy if you have
+direct control over the SQL string, but since Sequel abstracts that, the types
+have to be specified another way.  This is done by adding a __* suffix to the
+placeholder symbol (e.g. :$name__text, which will be compiled to "$1::text"
+in the SQL).  Prepared statements are always server side.
+
+=== SQLite
+
+SQLite supports bound variables and prepared statements exactly the same, since
+a new native prepared statement is created and executed for each call.
+
+=== MySQL
+
+The MySQL ruby driver does not support bound variables, so the the bound
+variable methods fall back to string interpolation.  It uses server side
+prepared statements.
+
+=== JDBC
+
+JDBC supports both prepared statements and bound variables.  Whether these
+are server side or client side depends on the JDBC driver.  For PostgreSQL
+over JDBC, you can add the prepareThreshold=N parameter to the connection
+string, which will use a server side prepared statement after N calls to
+the prepared statement.
+
+=== All Others
+
+Support is emulated using interpolation.

data/doc/sharding.rdoc

@@ -0,0 +1,113 @@
+= Read-Only Slaves/Writable Master and Database Sharding
+
+Starting with version 2.4.0, Sequel has support for read only slave databases
+with a writable master database, as well as database sharding (where you can
+pick a database connection to use for a given dataset).  Support for both
+features is database independent, and should work for all database adapters
+included with Sequel.
+
+== The :servers Database option
+
+Both features use the new :servers Database option.  The :servers option should
+be a hash with symbol keys and values that are either hashes or procs that
+return hashes.  Note that all servers should have the same schema, unless you
+really know what you are doing.
+
+== Master and Slave Database Configurations
+
+=== Single Read-Only Slave, Single Master
+
+To use a single, read-only slave that handles SELECT queries, the following
+is the simplest configuration:
+
+  DB=Sequel.connect('postgres://master_server/database', \
+    :servers=>{:read_only=>{:host=>'slave_server'}})
+
+This will use the host slave_server for SELECT queries and master_server for
+other queries.
+
+=== Multiple Read-Only Slaves, Single Master
+
+Let's say you have 4 slave database servers with names slave_server0,
+slave_server1, slave_server2, and slave_server3.
+
+  DB=Sequel.connect('postgres://master_server/database', \
+    :servers=>{:read_only=>proc{|db| :host=>db.get_slave_host}})
+  def DB.get_slave_host
+    @current_host ||= -1
+    "slave_server#{(@current_host+=1)%4}"
+  end
+
+This will use one of the slave servers for SELECT queries and use the
+master_server for other queries.  It's also possible to pick a random host
+instead of using the round robin approach presented above, but that can result
+in less optimal resource usage.
+
+=== Multiple Read-Only Slaves, Multiple Masters
+
+This involves the same basic idea as the multiple slaves, single master, but
+it shows that the master database is named :default.  So for 4 masters and
+4 slaves:
+
+  DB=Sequel.connect('postgres://master_server/database', \
+    :servers=>{:read_only=>proc{|db| :host=>db.get_slave_host}, \
+      :default=>proc{|db| :host=>db.get_master_host}})
+  def DB.get_slave_host
+    @current_slave_host ||= -1
+    "slave_server#{(@current_slave_host+=1)%4}"
+  end
+  def DB.get_master_host
+    @current_master_host ||= -1
+    "master_server#{(@current_master_host+=1)%4}"
+  end
+  
+== Sharding
+
+There is specific support in Sequel for handling master/slave database
+combinations, with the only necessary setup being the database configuration.
+However, since sharding is always going to be implementation dependent, Sequel
+supplies the basic infrastructure, but you have to tell it which server to use
+for each dataset.  Let's assume the simple scenario, a distributed rainbow
+table for SHA-1 hashes, sharding based on the first hex character (for a total
+of 16 shards).  First, you need to configure the database:
+
+  servers = {}
+  (('0'..'9').to_a + ('a'..'f').to_a).each do |hex|
+    servers[hex.to_sym] = {:host=>"hash_host_#{hex}"}
+  end
+  DB=Sequel.connect('postgres://hash_host/hashes', :servers=>servers)
+  
+This configures 17 servers, the 16 shard servers (/hash_host_[0-9a-f]/), and 1
+default server which will be used if no shard is specified ("hash_host").  If
+you want the default server to be one of the shard servers (e.g. hash_host_a),
+it's easiest to do:
+
+  DB=Sequel.connect('postgres://hash_host_a/hashes', :servers=>servers)
+
+That will still set up a second pool of connections for the default server,
+since it considers the default server and shard servers independent.  Note that
+if you always set the shard on a dataset before using it in queries, it will
+not attempt to connect to the default server.  Sequel may use the default
+server in queries it generates itself, such as to get column names or table
+schemas, so it is always good to have a default server that works.
+
+To set the shard for a given query, you use the Dataset#server method:
+
+  DB[:hashes].server(:a).filter(:hash=>/31337/)
+  
+That will return all matching rows on the hash_host_a shard that have a hash
+column that contains 31337.
+
+Rainbow tables are generally used to find specific hashes, so to save some
+work, you might want to add a method to the dataset that automatically sets
+the shard to use.  This is fairly easy using a Sequel::Model:
+
+  class Rainbow < Sequel::Model(:hashes)
+    def_dataset_method(:plaintext_for_hash) do |hash|
+      raise(ArgumentError, 'Invalid SHA-1 Hash') unless /\A[0-9a-f]{40}\z/.match(hash)
+      row = self.server(hash[0...1].to_sym).first(:hash=>hash)
+      row[:plaintext] if row
+    end
+  end
+  
+  Rainbow.plaintext_for_hash("e580726d31f6e1ad216ffd87279e536d1f74e606")

data/lib/sequel_core/adapters/ado.rb

@@ -13,15 +13,16 @@ module Sequel
   module ADO
     class Database < Sequel::Database
       set_adapter_scheme :ado
-      
-      AUTO_INCREMENT = 'IDENTITY(1,1)'.freeze
-      
-      def auto_increment_sql
-        AUTO_INCREMENT
-      end
-      
-      def connect
-        s = "driver=#{@opts[:driver] || 'SQL Server'};server=#{@opts[:host]};database=#{@opts[:database]}#{";uid=#{@opts[:user]};pwd=#{@opts[:password]}" if @opts[:user]}"
+
+      def connect(server)
+        opts = server_opts(server)
+        opts[:driver] ||= 'SQL Server'
+        case opts[:driver]
+        when 'SQL Server'
+          require 'sequel_core/adapters/shared/mssql'
+          extend Sequel::MSSQL::DatabaseMethods
+        end
+        s = "driver=#{opts[:driver]};server=#{opts[:host]};database=#{opts[:database]}#{";uid=#{opts[:user]};pwd=#{opts[:password]}" if opts[:user]}"
         handle = WIN32OLE.new('ADODB.Connection')
         handle.Open(s)
         handle
@@ -35,11 +36,14 @@ module Sequel
         ADO::Dataset.new(self, opts)
       end
     
-      def execute(sql)
+      def execute(sql, opts={})
         log_info(sql)
-        @pool.hold {|conn| conn.Execute(sql)}
+        synchronize(opts[:server]) do |conn|
+          r = conn.Execute(sql)
+          yield(r) if block_given?
+          r
+        end
       end
-      
       alias_method :do, :execute
     end
     
@@ -55,11 +59,12 @@ module Sequel
         end
       end
 
-      def fetch_rows(sql, &block)
-        @db.synchronize do
-          s = @db.execute sql
-          
-          @columns = s.Fields.extend(Enumerable).map {|x| x.Name.to_sym}
+      def fetch_rows(sql)
+        execute(sql) do |s|
+          @columns = s.Fields.extend(Enumerable).map do |column|
+            name = column.Name.empty? ? '(no column name)' : column.Name
+            name.to_sym
+          end
           
           unless s.eof
             s.moveFirst
@@ -69,6 +74,8 @@ module Sequel
         self
       end
       
+      private
+      
       def hash_row(row)
         @columns.inject({}) do |m, c|
           m[c] = row.shift

data/lib/sequel_core/adapters/db2.rb

@@ -6,29 +6,15 @@ module Sequel
       set_adapter_scheme :db2
       include DB2CLI
       
-      # AUTO_INCREMENT = 'IDENTITY(1,1)'.freeze
-      # 
-      # def auto_increment_sql
-      #   AUTO_INCREMENT
-      # end
-      
-      def check_error(rc, msg)
-        case rc
-        when SQL_SUCCESS, SQL_SUCCESS_WITH_INFO
-          nil
-        else
-          raise Error, msg
-        end
-      end
-      
       rc, @@env = SQLAllocHandle(SQL_HANDLE_ENV, SQL_NULL_HANDLE)
-      check_error(rc, "Could not allocate DB2 environment")
+      #check_error(rc, "Could not allocate DB2 environment")
 
-      def connect
+      def connect(server)
+        opts = server_opts(server)
         rc, dbc = SQLAllocHandle(SQL_HANDLE_DBC, @@env) 
         check_error(rc, "Could not allocate database connection")
         
-        rc = SQLConnect(dbc, @opts[:database], @opts[:user], @opts[:password]) 
+        rc = SQLConnect(dbc, opts[:database], opts[:user], opts[:password]) 
         check_error(rc, "Could not connect to database")
         
         dbc
@@ -44,8 +30,8 @@ module Sequel
         end
       end
     
-      def test_connection
-        @pool.hold {|conn|}
+      def test_connection(server=nil)
+        synchronize(server){|conn|}
         true
       end
 
@@ -53,9 +39,9 @@ module Sequel
         DB2::Dataset.new(self, opts)
       end
       
-      def execute(sql, &block)
+      def execute(sql, opts={})
         log_info(sql)
-        @pool.hold do |conn|
+        synchronize(opts[:server]) do |conn|
           rc, sth = SQLAllocHandle(SQL_HANDLE_STMT, @handle) 
           check_error(rc, "Could not allocate statement")
 
@@ -63,7 +49,7 @@ module Sequel
             rc = SQLExecDirect(sth, sql) 
             check_error(rc, "Could not execute statement")
             
-            block[sth] if block
+            yield(sth) if block_given?
 
             rc, rpc = SQLRowCount(sth)
             check_error(rc, "Could not get RPC") 
@@ -75,9 +61,22 @@ module Sequel
         end
       end
       alias_method :do, :execute
+      
+      private
+      
+      def check_error(rc, msg)
+        case rc
+        when SQL_SUCCESS, SQL_SUCCESS_WITH_INFO
+          nil
+        else
+          raise Error, msg
+        end
+      end
     end
     
     class Dataset < Sequel::Dataset
+      MAX_COL_SIZE = 256
+      
       def literal(v)
         case v
         when Time
@@ -89,21 +88,19 @@ module Sequel
         end
       end
 
-      def fetch_rows(sql, &block)
-        @db.synchronize do
-          @db.execute(sql) do |sth|
-            @column_info = get_column_info(sth)
-            @columns = @column_info.map {|c| c[:name]}
-            while (rc = SQLFetch(@handle)) != SQL_NO_DATA_FOUND
-              @db.check_error(rc, "Could not fetch row")
-              yield hash_row(sth)
-            end
+      def fetch_rows(sql)
+        execute(sql) do |sth|
+          @column_info = get_column_info(sth)
+          @columns = @column_info.map {|c| c[:name]}
+          while (rc = SQLFetch(@handle)) != SQL_NO_DATA_FOUND
+            @db.check_error(rc, "Could not fetch row")
+            yield hash_row(sth)
           end
         end
         self
       end
       
-      MAX_COL_SIZE = 256
+      private
 
       def get_column_info(sth)
         rc, column_count = SQLNumResultCols(sth)