sequel 4.8.0 → 4.9.0

data/lib/sequel/adapters/jdbc/h2.rb

@@ -143,11 +143,8 @@ module Sequel
       # Dataset class for H2 datasets accessed via JDBC.
       class Dataset < JDBC::Dataset
         SELECT_CLAUSE_METHODS = clause_methods(:select, %w'select distinct columns from join where group having compounds order limit')
-        BITWISE_METHOD_MAP = {:& =>:BITAND, :| => :BITOR, :^ => :BITXOR}
         APOS = Dataset::APOS
         HSTAR = "H*".freeze
-        BITCOMP_OPEN = "((0 - ".freeze
-        BITCOMP_CLOSE = ") - 1)".freeze
         ILIKE_PLACEHOLDER = ["CAST(".freeze, " AS VARCHAR_IGNORECASE)".freeze].freeze
         TIME_FORMAT = "'%H:%M:%S'".freeze
         
@@ -156,16 +153,8 @@ module Sequel
           case op
           when :ILIKE, :"NOT ILIKE"
             super(sql, (op == :ILIKE ? :LIKE : :"NOT LIKE"), [SQL::PlaceholderLiteralString.new(ILIKE_PLACEHOLDER, [args.at(0)]), args.at(1)])
-          when :&, :|, :^
-            sql << complex_expression_arg_pairs(args){|a, b| literal(SQL::Function.new(BITWISE_METHOD_MAP[op], a, b))}
-          when :<<
-            sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} * POWER(2, #{literal(b)}))"}
-          when :>>
-            sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} / POWER(2, #{literal(b)}))"}
-          when :'B~'
-            sql << BITCOMP_OPEN
-            literal_append(sql, args.at(0))
-            sql << BITCOMP_CLOSE
+          when :&, :|, :^, :<<, :>>, :'B~'
+            complex_expression_emulate_append(sql, op, args)
           else
             super
           end

data/lib/sequel/adapters/jdbc/hsqldb.rb

@@ -111,7 +111,6 @@ module Sequel
       
       # Dataset class for HSQLDB datasets accessed via JDBC.
       class Dataset < JDBC::Dataset
-        BITWISE_METHOD_MAP = {:& =>:BITAND, :| => :BITOR, :^ => :BITXOR}
         BOOL_TRUE = 'TRUE'.freeze
         BOOL_FALSE = 'FALSE'.freeze
         # HSQLDB does support common table expressions, but the support is broken.
@@ -124,8 +123,6 @@ module Sequel
         APOS = Dataset::APOS
         HSTAR = "H*".freeze
         BLOB_OPEN = "X'".freeze
-        BITCOMP_OPEN = "((0 - ".freeze
-        BITCOMP_CLOSE = ") - 1)".freeze
         DEFAULT_FROM = " FROM (VALUES (0))".freeze
         TIME_FORMAT = "'%H:%M:%S'".freeze
 
@@ -134,19 +131,8 @@ module Sequel
           case op
           when :ILIKE, :"NOT ILIKE"
             super(sql, (op == :ILIKE ? :LIKE : :"NOT LIKE"), args.map{|v| SQL::Function.new(:ucase, v)})
-          when :&, :|, :^
-            op = BITWISE_METHOD_MAP[op]
-            sql << complex_expression_arg_pairs(args){|a, b| literal(SQL::Function.new(op, a, b))}
-          when :<<
-            sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} * POWER(2, #{literal(b)}))"}
-          when :>>
-            sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} / POWER(2, #{literal(b)}))"}
-          when :%
-            sql << complex_expression_arg_pairs(args){|a, b| "MOD(#{literal(a)}, #{literal(b)})"}
-          when :'B~'
-            sql << BITCOMP_OPEN
-            literal_append(sql, args.at(0))
-            sql << BITCOMP_CLOSE
+          when :&, :|, :^, :%, :<<, :>>, :'B~'
+            complex_expression_emulate_append(sql, op, args)
           else
             super
           end

data/lib/sequel/adapters/mysql2.rb

@@ -144,6 +144,7 @@ module Sequel
     class Dataset < Sequel::Dataset
       include Sequel::MySQL::DatasetMethods
       include Sequel::MySQL::PreparedStatements::DatasetMethods
+      STREAMING_SUPPORTED = ::Mysql2::VERSION >= '0.3.12'
 
       Database::DatasetClass = self
 
@@ -160,9 +161,18 @@ module Sequel
         self
       end
 
+      # Use streaming to implement paging if Mysql2 supports it.
+      def paged_each(opts=OPTS, &block)
+        if STREAMING_SUPPORTED
+          stream.each(&block)
+        else
+          super
+        end
+      end
+
       # Return a clone of the dataset that will stream rows when iterating
       # over the result set, so it can handle large datasets that
-      # won't fit in memory (Requires mysql 0.3.12 to have an effect).
+      # won't fit in memory (Requires mysql 0.3.12+ to have an effect).
       def stream
         clone(:stream=>true)
       end

data/lib/sequel/adapters/postgres.rb

@@ -618,6 +618,7 @@ module Sequel
 
       Database::DatasetClass = self
       APOS = Sequel::Dataset::APOS
+      DEFAULT_CURSOR_NAME = 'sequel_cursor'.freeze
       
       # Yield all rows returned by executing the given SQL and converting
       # the types.
@@ -626,15 +627,22 @@ module Sequel
         execute(sql){|res| yield_hash_rows(res, fetch_rows_set_cols(res)){|h| yield h}}
       end
       
+      # Use a cursor for paging.
+      def paged_each(opts=OPTS, &block)
+        use_cursor(opts).each(&block)
+      end
+
       # Uses a cursor for fetching records, instead of fetching the entire result
       # set at once.  Can be used to process large datasets without holding
-      # all rows in memory (which is what the underlying drivers do
+      # all rows in memory (which is what the underlying drivers may do
       # by default). Options:
       #
-      # * :rows_per_fetch - the number of rows per fetch (default 1000).  Higher
-      #   numbers result in fewer queries but greater memory use.
-      # * :cursor_name - the name assigned to the cursor (default 'sequel_cursor').
-      #   Nested cursors require different names.
+      # :cursor_name :: The name assigned to the cursor (default 'sequel_cursor').
+      #                 Nested cursors require different names.
+      # :hold :: Declare the cursor WITH HOLD and don't use transaction around the
+      #          cursor usage.
+      # :rows_per_fetch :: The number of rows per fetch (default 1000).  Higher
+      #                    numbers result in fewer queries but greater memory use.
       #
       # Usage:
       #
@@ -645,7 +653,19 @@ module Sequel
       # This is untested with the prepared statement/bound variable support,
       # and unlikely to work with either.
       def use_cursor(opts=OPTS)
-        clone(:cursor=>{:rows_per_fetch=>1000, :cursor_name => 'sequel_cursor'}.merge(opts))
+        clone(:cursor=>{:rows_per_fetch=>1000}.merge(opts))
+      end
+
+      # Replace the WHERE clause with one that uses CURRENT OF with the given
+      # cursor name (or the default cursor name).  This allows you to update a
+      # large dataset by updating individual rows while processing the dataset
+      # via a cursor:
+      #
+      #   DB[:huge_table].use_cursor(:rows_per_fetch=>1).each do |row|
+      #     DB[:huge_table].where_current_of.update(:column=>ruby_method(row))
+      #   end
+      def where_current_of(cursor_name=DEFAULT_CURSOR_NAME)
+        clone(:where=>Sequel.lit(['CURRENT OF '], Sequel.identifier(cursor_name)))
       end
 
       if SEQUEL_POSTGRES_USES_PG
@@ -760,11 +780,14 @@ module Sequel
       # Use a cursor to fetch groups of records at a time, yielding them to the block.
       def cursor_fetch_rows(sql)
         server_opts = {:server=>@opts[:server] || :read_only}
-        cursor_name = quote_identifier(@opts[:cursor][:cursor_name])
-        db.transaction(server_opts) do 
+        cursor = @opts[:cursor]
+        hold = cursor[:hold]
+        cursor_name = quote_identifier(cursor[:cursor_name] || DEFAULT_CURSOR_NAME)
+        rows_per_fetch = cursor[:rows_per_fetch].to_i
+
+        db.send(*(hold ? [:synchronize, server_opts[:server]] : [:transaction, server_opts])) do 
           begin
-            execute_ddl("DECLARE #{cursor_name} NO SCROLL CURSOR WITHOUT HOLD FOR #{sql}", server_opts)
-            rows_per_fetch = @opts[:cursor][:rows_per_fetch].to_i
+            execute_ddl("DECLARE #{cursor_name} NO SCROLL CURSOR WITH#{'OUT' unless hold} HOLD FOR #{sql}", server_opts)
             rows_per_fetch = 1000 if rows_per_fetch <= 0
             fetch_sql = "FETCH FORWARD #{rows_per_fetch} FROM #{cursor_name}"
             cols = nil

data/lib/sequel/adapters/shared/db2.rb

@@ -256,16 +256,8 @@ module Sequel
 
       def complex_expression_sql_append(sql, op, args)
         case op
-        when :&, :|, :^
-          # works with db2 v9.5 and after
-          op = BITWISE_METHOD_MAP[op]
-          sql << complex_expression_arg_pairs(args){|a, b| literal(SQL::Function.new(op, a, b))}
-        when :<<
-          sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} * POWER(2, #{literal(b)}))"}
-        when :>>
-          sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} / POWER(2, #{literal(b)}))"}
-        when :%
-          sql << complex_expression_arg_pairs(args){|a, b| "MOD(#{literal(a)}, #{literal(b)})"}
+        when :&, :|, :^, :%, :<<, :>>
+          complex_expression_emulate_append(sql, op, args)
         when :'B~'
           literal_append(sql, SQL::Function.new(:BITNOT, *args))
         when :extract

data/lib/sequel/adapters/shared/mssql.rb

@@ -523,19 +523,21 @@ module Sequel
         when :'||'
           super(sql, :+, args)
         when :LIKE, :"NOT LIKE"
-          super(sql, op, args.map{|a| LiteralString.new("(#{literal(a)} COLLATE #{CASE_SENSITIVE_COLLATION})")})
+          super(sql, op, args.map{|a| Sequel.lit(["(", " COLLATE #{CASE_SENSITIVE_COLLATION})"], a)})
         when :ILIKE, :"NOT ILIKE"
-          super(sql, (op == :ILIKE ? :LIKE : :"NOT LIKE"), args.map{|a| LiteralString.new("(#{literal(a)} COLLATE #{CASE_INSENSITIVE_COLLATION})")})
-        when :<<
-          sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} * POWER(2, #{literal(b)}))"}
-        when :>>
-          sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} / POWER(2, #{literal(b)}))"}
+          super(sql, (op == :ILIKE ? :LIKE : :"NOT LIKE"), args.map{|a| Sequel.lit(["(", " COLLATE #{CASE_INSENSITIVE_COLLATION})"], a)})
+        when :<<, :>>
+          complex_expression_emulate_append(sql, op, args)
         when :extract
           part = args.at(0)
           raise(Sequel::Error, "unsupported extract argument: #{part.inspect}") unless format = EXTRACT_MAP[part]
           if part == :second
-            expr = literal(args.at(1))
-            sql << DATEPART_SECOND_OPEN << format.to_s << COMMA << expr << DATEPART_SECOND_MIDDLE << expr << DATEPART_SECOND_CLOSE
+            expr = args.at(1)
+            sql << DATEPART_SECOND_OPEN << format.to_s << COMMA
+            literal_append(sql, expr)
+            sql << DATEPART_SECOND_MIDDLE
+            literal_append(sql, expr)
+            sql << DATEPART_SECOND_CLOSE
           else
             sql << DATEPART_OPEN << format.to_s << COMMA
             literal_append(sql, args.at(1))

data/lib/sequel/adapters/shared/oracle.rb

@@ -254,43 +254,28 @@ module Sequel
       APOS_RE = Dataset::APOS_RE
       DOUBLE_APOS = Dataset::DOUBLE_APOS
       FROM = Dataset::FROM
-      BITCOMP_OPEN = "((0 - ".freeze
-      BITCOMP_CLOSE = ") - 1)".freeze
       TIMESTAMP_FORMAT = "TIMESTAMP '%Y-%m-%d %H:%M:%S%N %z'".freeze
       TIMESTAMP_OFFSET_FORMAT = "%+03i:%02i".freeze
       BOOL_FALSE = "'N'".freeze
       BOOL_TRUE = "'Y'".freeze
       HSTAR = "H*".freeze
       DUAL = ['DUAL'.freeze].freeze
+      BITAND_PROC = lambda{|a, b| Sequel.lit(["CAST(BITAND(", ", ", ") AS INTEGER)"], a, b)}
 
       def complex_expression_sql_append(sql, op, args)
         case op
         when :&
-          sql << complex_expression_arg_pairs(args){|a, b| "CAST(BITAND(#{literal(a)}, #{literal(b)}) AS INTEGER)"}
+          complex_expression_arg_pairs_append(sql, args, &BITAND_PROC)
         when :|
-          sql << complex_expression_arg_pairs(args) do |a, b|
-            s1 = ''
-            complex_expression_sql_append(s1, :&, [a, b])
-            "(#{literal(a)} - #{s1} + #{literal(b)})"
-          end
+          complex_expression_arg_pairs_append(sql, args){|a, b| Sequel.lit(["(", " - ", " + ", ")"], a, complex_expression_arg_pairs([a, b], &BITAND_PROC), b)}
         when :^
-          sql << complex_expression_arg_pairs(args) do |*x|
-            s1 = ''
-            s2 = ''
-            complex_expression_sql_append(s1, :|, x)
-            complex_expression_sql_append(s2, :&, x)
-            "(#{s1} - #{s2})"
+          complex_expression_arg_pairs_append(sql, args) do |*x|
+            s1 = complex_expression_arg_pairs(x){|a, b| Sequel.lit(["(", " - ", " + ", ")"], a, complex_expression_arg_pairs([a, b], &BITAND_PROC), b)}
+            s2 = complex_expression_arg_pairs(x, &BITAND_PROC)
+            Sequel.lit(["(", " - ", ")"], s1, s2)
           end
-        when :'B~'
-          sql << BITCOMP_OPEN
-          literal_append(sql, args.at(0))
-          sql << BITCOMP_CLOSE
-        when :<<
-          sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} * power(2, #{literal(b)}))"}
-        when :>>
-          sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} / power(2, #{literal(b)}))"}
-        when :%
-          sql << complex_expression_arg_pairs(args){|a, b| "MOD(#{literal(a)}, #{literal(b)})"}
+        when :%, :<<, :>>, :'B~'
+          complex_expression_emulate_append(sql, op, args)
         else
           super
         end

data/lib/sequel/adapters/shared/postgres.rb

@@ -1183,6 +1183,24 @@ module Sequel
         end
       end
 
+      # Disables automatic use of INSERT ... RETURNING.  You can still use
+      # returning manually to force the use of RETURNING when inserting.
+      #
+      # This is designed for cases where INSERT RETURNING cannot be used,
+      # such as when you are using partitioning with trigger functions
+      # or conditional rules, or when you are using a PostgreSQL version
+      # less than 8.2, or a PostgreSQL derivative that does not support
+      # returning.
+      #
+      # Note that when this method is used, insert will not return the
+      # primary key of the inserted row, you will have to get the primary
+      # key of the inserted row before inserting via nextval, or after
+      # inserting via currval or lastval (making sure to use the same
+      # database connection for currval or lastval).
+      def disable_insert_returning
+        clone(:disable_insert_returning=>true)
+      end
+
       # Return the results of an EXPLAIN query as a string
       def explain(opts=OPTS)
         with_sql((opts[:analyze] ? EXPLAIN_ANALYZE : EXPLAIN) + select_sql).map(QUERY_PLAN).join(CRLF)
@@ -1220,24 +1238,24 @@ module Sequel
       # Insert given values into the database.
       def insert(*values)
         if @opts[:returning]
-          # already know which columns to return, let the standard code
-          # handle it
+          # Already know which columns to return, let the standard code handle it
           super
-        elsif @opts[:sql]
-          # raw SQL used, so don't know which table is being inserted
-          # into, and therefore can't determine primary key.  Run the
-          # insert statement and return nil.
+        elsif @opts[:sql] || @opts[:disable_insert_returning]
+          # Raw SQL used or RETURNING disabled, just use the default behavior
+          # and return nil since sequence is not known.
           super
           nil
         else
-          # Force the use of RETURNING with the primary key value.
+          # Force the use of RETURNING with the primary key value,
+          # unless it has been disabled.
           returning(insert_pk).insert(*values){|r| return r.values.first}
         end
       end
 
-      # Insert a record returning the record inserted
+      # Insert a record returning the record inserted.  Always returns nil without
+      # inserting a query if disable_insert_returning is used.
       def insert_select(*values)
-        returning.insert(*values){|r| return r}
+        returning.insert(*values){|r| return r} unless @opts[:disable_insert_returning]
       end
 
       # Locks all tables in the dataset's FROM clause (but not in JOINs) with
@@ -1279,6 +1297,11 @@ module Sequel
         true
       end
 
+      # True unless insert returning has been disabled for this dataset.
+      def supports_insert_select?
+        !@opts[:disable_insert_returning]
+      end
+
       # PostgreSQL 9.3rc1+ supports lateral subqueries
       def supports_lateral_subqueries?
         server_version >= 90300

data/lib/sequel/adapters/shared/sqlanywhere.rb

@@ -306,10 +306,8 @@ module Sequel
         case op
         when :'||'
           super(sql, :+, args)
-        when :<<
-          sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} * POWER(2, #{literal(b)}))"}
-        when :>>
-          sql << complex_expression_arg_pairs(args){|a, b| "(#{literal(a)} / POWER(2, #{literal(b)}))"}
+        when :<<, :>>
+          complex_expression_emulate_append(sql, op, args)
         when :LIKE, :"NOT LIKE"
           sql << Sequel::Dataset::PAREN_OPEN
           literal_append(sql, args.at(0))

data/lib/sequel/adapters/shared/sqlite.rb

@@ -332,8 +332,9 @@ module Sequel
       end
 
       DATABASE_ERROR_REGEXPS = {
-        /(is|are) not unique\z|UNIQUE constraint failed: .+\z/ => UniqueConstraintViolation,
-        /foreign key constraint failed\z|FOREIGN KEY constraint failed\z/ => ForeignKeyConstraintViolation,
+        /(is|are) not unique\z|PRIMARY KEY must be unique\z|UNIQUE constraint failed: .+\z/ => UniqueConstraintViolation,
+        /foreign key constraint failed\z/i => ForeignKeyConstraintViolation,
+        /\ACHECK constraint failed/ => CheckConstraintViolation,
         /\A(SQLITE ERROR 19 \(CONSTRAINT\) : )?constraint failed\z/ => ConstraintViolation,
         /may not be NULL\z|NOT NULL constraint failed: .+\z/ => NotNullConstraintViolation,
       }.freeze
@@ -539,11 +540,7 @@ module Sequel
           sql << NOT_SPACE
           complex_expression_sql_append(sql, (op == :"NOT ILIKE" ? :ILIKE : :LIKE), args)
         when :^
-          sql << complex_expression_arg_pairs(args) do |a, b|
-            a = literal(a)
-            b = literal(b)
-            "((~(#{a} & #{b})) & (#{a} | #{b}))"
-          end
+          complex_expression_arg_pairs_append(sql, args){|a, b| Sequel.lit(["((~(", " & ", ")) & (", " | ", "))"], a, b, a, b)}
         when :extract
           part = args.at(0)
           raise(Sequel::Error, "unsupported extract argument: #{part.inspect}") unless format = EXTRACT_MAP[part]

data/lib/sequel/database/schema_methods.rb

@@ -134,6 +134,21 @@ module Sequel
       end
     end
 
+    # Forcibly create a join table, attempting to drop it if it already exists, then creating it.
+    def create_join_table!(hash, options=OPTS)
+      drop_table?(join_table_name(hash, options))
+      create_join_table(hash, options)
+    end
+    
+    # Creates the join table unless it already exists.
+    def create_join_table?(hash, options=OPTS)
+      if supports_create_table_if_not_exists?
+        create_join_table(hash, options.merge(:if_not_exists=>true))
+      elsif !table_exists?(join_table_name(hash, options))
+        create_join_table(hash, options)
+      end
+    end
+
     # Creates a table with the columns given in the provided block:
     #
     #   DB.create_table :posts do

data/lib/sequel/dataset.rb

@@ -36,5 +36,5 @@ module Sequel
     include SQL::StringMethods
   end
   
-  require(%w"query actions features graph prepared_statements misc mutation sql", 'dataset')
+  require(%w"query actions features graph prepared_statements misc mutation sql placeholder_literalizer", 'dataset')
 end

data/lib/sequel/dataset/actions.rb

@@ -42,11 +42,7 @@ module Sequel
     #   # Iterate over all rows in the table
     #   DB[:table].all{|row| p row}
     def all(&block)
-      a = []
-      each{|r| a << r}
-      post_load(a)
-      a.each(&block) if block
-      a
+      _all(block){|a| each{|r| a << r}}
     end
     
     # Returns the average value for the given column/expression.
@@ -434,49 +430,107 @@ module Sequel
       import(columns, hashes.map{|h| columns.map{|c| h[c]}}, opts)
     end
 
-    # Yields each row in the dataset, but interally uses multiple queries as needed with
-    # limit and offset to process the entire result set without keeping all
-    # rows in the dataset in memory, even if the underlying driver buffers all
-    # query results in memory.
+    # Yields each row in the dataset, but interally uses multiple queries as needed to
+    # process the entire result set without keeping all rows in the dataset in memory,
+    # even if the underlying driver buffers all query results in memory.
     #
     # Because this uses multiple queries internally, in order to remain consistent,
-    # it also uses a transaction internally.  Additionally, to make sure that all rows
-    # in the dataset are yielded and none are yielded twice, the dataset must have an
-    # unambiguous order.  Sequel requires that datasets using this method have an
-    # order, but it cannot ensure that the order is unambiguous.
+    # it also uses a transaction internally.  Additionally, to work correctly, the dataset
+    # must have unambiguous order.  Using an ambiguous order can result in an infinite loop,
+    # as well as subtler bugs such as yielding duplicate rows or rows being skipped.
+    #
+    # Sequel checks that the datasets using this method have an order, but it cannot
+    # ensure that the order is unambiguous.
     #
     # Options:
     # :rows_per_fetch :: The number of rows to fetch per query.  Defaults to 1000.
+    # :strategy :: The strategy to use for paging of results.  By default this is :offset,
+    #              for using an approach with a limit and offset for every page.  This can
+    #              be set to :filter, which uses a limit and a filter that excludes
+    #              rows from previous pages.  In order for this strategy to work, you must be
+    #              selecting the columns you are ordering by, and non of the columns can contain
+    #              NULLs.  Note that some Sequel adapters have optimized implementations that will
+    #              use cursors or streaming regardless of the :strategy option used.
+    # :filter_values :: If the :strategy=>:filter option is used, this option should be a proc
+    #                   that accepts the last retreived row for the previous page and an array of
+    #                   ORDER BY expressions, and returns an array of values relating to those
+    #                   expressions for the last retrieved row.  You will need to use this option
+    #                   if your ORDER BY expressions are not simple columns, if they contain
+    #                   qualified identifiers that would be ambiguous unqualified, if they contain
+    #                   any identifiers that are aliased in SELECT, and potentially other cases.
+    #
+    # Examples:
+    #
+    #   DB[:table].order(:id).paged_each{|row| ...}
+    #   # SELECT * FROM table ORDER BY id LIMIT 1000
+    #   # SELECT * FROM table ORDER BY id LIMIT 1000 OFFSET 1000
+    #   # ...
+    #
+    #   DB[:table].order(:id).paged_each(:rows_per_fetch=>100){|row| ...}
+    #   # SELECT * FROM table ORDER BY id LIMIT 100
+    #   # SELECT * FROM table ORDER BY id LIMIT 100 OFFSET 100
+    #   # ...
+    #
+    #   DB[:table].order(:id).paged_each(:strategy=>:filter){|row| ...}
+    #   # SELECT * FROM table ORDER BY id LIMIT 1000
+    #   # SELECT * FROM table WHERE id > 1001 ORDER BY id LIMIT 1000
+    #   # ...
+    #
+    #   DB[:table].order(:table__id).paged_each(:strategy=>:filter,
+    #     :filter_values=>proc{|row, exprs| [row[:id]]}){|row| ...}
+    #   # SELECT * FROM table ORDER BY id LIMIT 1000
+    #   # SELECT * FROM table WHERE id > 1001 ORDER BY id LIMIT 1000
+    #   # ...
     def paged_each(opts=OPTS)
       unless @opts[:order]
         raise Sequel::Error, "Dataset#paged_each requires the dataset be ordered"
       end
 
       total_limit = @opts[:limit]
-      offset = @opts[:offset] || 0
-
+      offset = @opts[:offset]
       if server = @opts[:server]
         opts = opts.merge(:server=>server)
       end
 
       rows_per_fetch = opts[:rows_per_fetch] || 1000
-      num_rows_yielded = rows_per_fetch
-      total_rows = 0
+      strategy = if offset || total_limit
+        :offset
+      else
+        opts[:strategy] || :offset
+      end
 
       db.transaction(opts) do
-        while num_rows_yielded == rows_per_fetch && (total_limit.nil? || total_rows < total_limit)
-          if total_limit && total_rows + rows_per_fetch > total_limit
-            rows_per_fetch = total_limit - total_rows
+        case strategy
+        when :filter
+          filter_values = opts[:filter_values] || proc{|row, exprs| exprs.map{|e| row[hash_key_symbol(e)]}}
+          base_ds = ds = limit(rows_per_fetch)
+          while ds
+            last_row = nil
+            ds.each do |row|
+              last_row = row
+              yield row
+            end
+            ds = (base_ds.where(ignore_values_preceding(last_row, &filter_values)) if last_row)
           end
+        else
+          offset ||= 0
+          num_rows_yielded = rows_per_fetch
+          total_rows = 0
 
-          num_rows_yielded = 0
-          limit(rows_per_fetch, offset).each do |row|
-            num_rows_yielded += 1
-            total_rows += 1 if total_limit
-            yield row
-          end
+          while num_rows_yielded == rows_per_fetch && (total_limit.nil? || total_rows < total_limit)
+            if total_limit && total_rows + rows_per_fetch > total_limit
+              rows_per_fetch = total_limit - total_rows
+            end
+
+            num_rows_yielded = 0
+            limit(rows_per_fetch, offset).each do |row|
+              num_rows_yielded += 1
+              total_rows += 1 if total_limit
+              yield row
+            end
 
-          offset += rows_per_fetch
+            offset += rows_per_fetch
+          end
         end
       end
 
@@ -720,12 +774,54 @@ module Sequel
       end
     end
 
+    # Run the given SQL and return an array of all rows.  If a block is given,
+    # each row is yielded to the block after all rows are loaded. See with_sql_each.
+    def with_sql_all(sql, &block)
+      _all(block){|a| with_sql_each(sql){|r| a << r}}
+    end
+
     # Execute the given SQL and return the number of rows deleted.  This exists
     # solely as an optimization, replacing with_sql(sql).delete.  It's significantly
     # faster as it does not require cloning the current dataset.
     def with_sql_delete(sql)
       execute_dui(sql)
     end
+    alias with_sql_update with_sql_delete
+
+    # Run the given SQL and yield each returned row to the block.
+    #
+    # This method should not be called on a shared dataset if the columns selected
+    # in the given SQL do not match the columns in the receiver.
+    def with_sql_each(sql)
+      if row_proc = @row_proc
+        fetch_rows(sql){|r| yield row_proc.call(r)}
+      else
+        fetch_rows(sql){|r| yield r}
+      end
+      self
+    end
+    
+    # Run the given SQL and return the first row, or nil if no rows were returned.
+    # See with_sql_each.
+    def with_sql_first(sql)
+      with_sql_each(sql){|r| return r}
+      nil
+    end
+
+    # Run the given SQL and return the first value in the first row, or nil if no
+    # rows were returned.  For this to make sense, the SQL given should select
+    # only a single value.  See with_sql_each.
+    def with_sql_single_value(sql)
+      if r = with_sql_first(sql)
+        r.values.first
+      end
+    end
+
+    # Execute the given SQL and (on most databases) return the primary key of the
+    # inserted row.
+    def with_sql_insert(sql)
+      execute_insert(sql)
+    end
 
     protected
 
@@ -754,6 +850,15 @@ module Sequel
   
     private
     
+    # Internals of all and with_sql_all
+    def _all(block)
+      a = []
+      yield a
+      post_load(a)
+      a.each(&block) if block
+      a
+    end
+    
     # Internals of +select_hash+ and +select_hash_groups+
     def _select_hash(meth, key_column, value_column)
       select(*(key_column.is_a?(Array) ? key_column : [key_column]) + (value_column.is_a?(Array) ? value_column : [value_column])).
@@ -850,6 +955,33 @@ module Sequel
       s.is_a?(Array) ? s.map{|c| hash_key_symbol(c)} : hash_key_symbol(s)
     end
     
+    # Returns an expression that will ignore values preceding the given row, using the
+    # receiver's current order. This yields the row and the array of order expressions
+    # to the block, which should return an array of values to use.
+    def ignore_values_preceding(row)
+      @opts[:order].map{|v| v.is_a?(SQL::OrderedExpression) ? v.expression : v}
+
+      order_exprs = @opts[:order].map do |v|
+        if v.is_a?(SQL::OrderedExpression)
+          descending = v.descending
+          v = v.expression
+        else
+          descending = false
+        end
+        [v, descending]
+      end
+
+      row_values = yield(row, order_exprs.map{|e| e.first})
+
+      last_expr = []
+      cond = order_exprs.zip(row_values).map do |(v, descending), value|
+        expr =  last_expr + [SQL::BooleanExpression.new(descending ? :< : :>, v, value)]
+        last_expr += [SQL::BooleanExpression.new(:'=', v, value)]
+        Sequel.&(*expr)
+      end
+      Sequel.|(*cond)
+    end
+
     # Modify the identifier returned from the database based on the
     # identifier_output_method.
     def output_identifier(v)