sequel_core 1.0.9.1 → 1.0.10

data/CHANGELOG

@@ -1,4 +1,14 @@
-=== SVN
+=== 1.0.10 (2008-02-13)
+
+* Fixed Datset#group_and_count to work inside a query block (#152).
+
+* Changed datasets with transforms to automatically transform hash filters (#155).
+
+* Changed Marshal stock transform to use Base64 encoding with backward-compatibility to support existing marshaled values (#154).
+
+* Added support for inserting multiple records in a single statement using #multi_insert in MySQL adapter (#153).
+
+* Added support for :slice option (same as :commit_every) in Dataset#multi_insert.
 
 * Changed Dataset#all to accept opts and iteration block.
 

data/Rakefile

@@ -9,7 +9,7 @@ include FileUtils
 # Configuration
 ##############################################################################
 NAME = "sequel_core"
-VERS = "1.0.9.1"
+VERS = "1.0.10"
 CLEAN.include ["**/.*.sw?", "pkg/*", ".config", "doc/*", "coverage/*"]
 RDOC_OPTS = [
   "--quiet", 

data/lib/sequel_core/adapters/mysql.rb

@@ -357,6 +357,35 @@ module Sequel
         end
         self
       end
+      
+      def multi_insert_sql(keys, array)
+        values = array.map {|r| "(#{literal(keys.map {|k| r[k]})})"}.join(COMMA_SEPARATOR)
+        columns = keys.map {|c| literal(c)}.join(COMMA_SEPARATOR)
+        "INSERT INTO #{@opts[:from]} (#{columns}) VALUES #{values}"
+      end
+      
+      # Inserts multiple records into the associated table. This method can be
+      # to efficiently insert a large amounts of records into a table. Inserts
+      # are automatically wrapped in a transaction. If the :commit_every 
+      # option is specified, the method will generate a separate transaction 
+      # for each batch of records, e.g.:
+      #
+      #   dataset.multi_insert(list, :commit_every => 1000)
+      def multi_insert(list, opts = {})
+        keys = list.first.keys
+        
+        if every = (opts[:commit_every] || opts[:slice])
+          list.each_slice(every) do |s|
+            @db.transaction do
+              @db.execute(multi_insert_sql(keys, s))
+            end
+          end
+        else
+          @db.transaction do
+            @db.execute(multi_insert_sql(keys, list))
+          end
+        end
+      end
     end
   end
 end

data/lib/sequel_core/dataset/convenience.rb

@@ -221,12 +221,12 @@ module Sequel
       # Inserts multiple records into the associated table. This method can be
       # to efficiently insert a large amounts of records into a table. Inserts
       # are automatically wrapped in a transaction. If the :commit_every 
-      # option is specified, the method will generate a separate transaction 
-      # for each batch of records, e.g.:
+      # or :slice option is specified, the method will generate a separate 
+      # transaction for each batch of records, e.g.:
       #
       #   dataset.multi_insert(list, :commit_every => 1000)
       def multi_insert(list, opts = {})
-        if every = opts[:commit_every]
+        if every = (opts[:commit_every] || opts[:slice])
           list.each_slice(every) do |s|
             @db.transaction do
               s.each {|r| @db.execute(insert_sql(r))}
@@ -246,9 +246,10 @@ module Sequel
         def insert(*args); raise Error, "#insert cannot be invoked inside a query block."; end
         def update(*args); raise Error, "#update cannot be invoked inside a query block."; end
         def delete(*args); raise Error, "#delete cannot be invoked inside a query block."; end
-        
-        def clone(opts)
+
+        def clone(opts = nil)
           @opts.merge!(opts)
+          self
         end
       end
       

data/lib/sequel_core/dataset/sql.rb

@@ -265,8 +265,13 @@ module Sequel
         if cond === true || cond === false
           raise Error::InvalidFilter, "Invalid filter specified. Did you mean to supply a block?"
         end
+        
+        if cond.is_a?(Hash)
+          cond = transform_save(cond) if @transform
+          filter = cond
+        end
         parenthesize = !(cond.is_a?(Hash) || cond.is_a?(Array))
-        filter = cond.is_a?(Hash) && cond
+
         if !@opts[clause].nil? and @opts[clause].any?
           l = expression_list(@opts[clause])
           r = expression_list(block || cond, parenthesize)

data/lib/sequel_core/dataset.rb

@@ -1,6 +1,7 @@
 require 'time'
 require 'date'
 require 'yaml'
+require 'base64'
 
 require File.join(File.dirname(__FILE__), 'dataset/sql')
 require File.join(File.dirname(__FILE__), 'dataset/sequelizer')
@@ -298,8 +299,15 @@ module Sequel
     end
     
     STOCK_TRANSFORMS = {
-      :marshal => [proc {|v| Marshal.load(v)}, proc {|v| Marshal.dump(v)}],
-      :yaml => [proc {|v| YAML.load v if v}, proc {|v| v.to_yaml}]
+      :marshal => [
+        # for backwards-compatibility we support also non-base64-encoded values.
+        proc {|v| Marshal.load(Base64.decode64(v)) rescue Marshal.load(v)}, 
+        proc {|v| Base64.encode64(Marshal.dump(v))}
+      ],
+      :yaml => [
+        proc {|v| YAML.load v if v}, 
+        proc {|v| v.to_yaml}
+      ]
     }
     
     # Sets a value transform which is used to convert values loaded and saved

data/spec/adapters/mysql_spec.rb

@@ -264,7 +264,7 @@ end
 #   end
 # end
 # 
-context "Joiמed MySQL dataset" do
+context "Joined MySQL dataset" do
   setup do
     @ds = MYSQL_DB[:nodes].join(:attributes, :node_id => :id)
     @ds2 = MYSQL_DB[:nodes]
@@ -440,4 +440,101 @@ context "A MySQL database" do
     MYSQL_DB[:posts].full_text_search(:title, '+ruby -rails', :boolean => true).sql.should ==
       "SELECT * FROM posts WHERE (MATCH (`title`) AGAINST ('+ruby -rails' IN BOOLEAN MODE))"
   end
+end
+
+class Sequel::MySQL::Database
+  alias_method :orig_execute, :execute
+  attr_accessor :sqls
+  def execute(sql, &block)
+    @sqls ||= []; @sqls << sql
+    orig_execute(sql, &block)
+  end
+
+  def transaction
+    @pool.hold do |conn|
+      @transactions ||= []
+      if @transactions.include? Thread.current
+        return yield(conn)
+      end
+      @sqls ||= []; @sqls << SQL_BEGIN
+      conn.query(SQL_BEGIN)
+      begin
+        @transactions << Thread.current
+        result = yield(conn)
+        @sqls ||= []; @sqls << SQL_COMMIT
+        conn.query(SQL_COMMIT)
+        result
+      rescue => e
+        @sqls ||= []; @sqls << SQL_ROLLBACK
+        conn.query(SQL_ROLLBACK)
+        raise e unless Sequel::Error::Rollback === e
+      ensure
+        @transactions.delete(Thread.current)
+      end
+    end
+  end
+end
+
+context "MySQL::Dataset#multi_insert" do
+  setup do
+    @d = MYSQL_DB[:items]
+    @d.delete # remove all records
+    MYSQL_DB.sqls.clear
+  end
+  
+  specify "should insert multiple records in a single statement" do
+    @d.multi_insert([{:name => 'abc'}, {:name => 'def'}])
+    
+    MYSQL_DB.sqls.should == [
+      'BEGIN',
+      "INSERT INTO items (`name`) VALUES ('abc'), ('def')",
+      'COMMIT'
+    ]
+
+    @d.all.should == [
+      {:name => 'abc', :value => nil}, {:name => 'def', :value => nil}
+    ]
+  end
+
+  specify "should split the list of records into batches if :commit_every option is given" do
+    @d.multi_insert([{:value => 1}, {:value => 2}, {:value => 3}, {:value => 4}],
+      :commit_every => 2)
+
+    MYSQL_DB.sqls.should == [
+      'BEGIN',
+      "INSERT INTO items (`value`) VALUES (1), (2)",
+      'COMMIT',
+      'BEGIN',
+      "INSERT INTO items (`value`) VALUES (3), (4)",
+      'COMMIT'
+    ]
+    
+    @d.all.should == [
+      {:name => nil, :value => 1}, 
+      {:name => nil, :value => 2},
+      {:name => nil, :value => 3}, 
+      {:name => nil, :value => 4}
+    ]
+  end
+
+  specify "should split the list of records into batches if :slice option is given" do
+    @d.multi_insert([{:value => 1}, {:value => 2}, {:value => 3}, {:value => 4}],
+      :slice => 2)
+
+    MYSQL_DB.sqls.should == [
+      'BEGIN',
+      "INSERT INTO items (`value`) VALUES (1), (2)",
+      'COMMIT',
+      'BEGIN',
+      "INSERT INTO items (`value`) VALUES (3), (4)",
+      'COMMIT'
+    ]
+    
+    @d.all.should == [
+      {:name => nil, :value => 1}, 
+      {:name => nil, :value => 2},
+      {:name => nil, :value => 3}, 
+      {:name => nil, :value => 4}
+    ]
+  end
 end

data/spec/dataset_spec.rb

@@ -1032,11 +1032,18 @@ context "Dataset#group_and_count" do
   end
   
   specify "should format SQL properly" do
-    @ds.group_and_count(:name).sql.should == "SELECT name, count(*) AS count FROM test GROUP BY name ORDER BY count"
+    @ds.group_and_count(:name).sql.should == 
+      "SELECT name, count(*) AS count FROM test GROUP BY name ORDER BY count"
   end
 
   specify "should accept multiple columns for grouping" do
-    @ds.group_and_count(:a, :b).sql.should == "SELECT a, b, count(*) AS count FROM test GROUP BY a, b ORDER BY count"
+    @ds.group_and_count(:a, :b).sql.should == 
+      "SELECT a, b, count(*) AS count FROM test GROUP BY a, b ORDER BY count"
+  end
+  
+  specify "should work within query block" do
+    @ds.query{group_and_count(:a, :b)}.sql.should == 
+      "SELECT a, b, count(*) AS count FROM test GROUP BY a, b ORDER BY count"
   end
 end
 
@@ -2053,7 +2060,7 @@ context "Dataset#multi_insert" do
     ]
   end
   
-  specify "should accept the commit_every option for committing every x records" do
+  specify "should accept the :commit_every option for committing every x records" do
     @ds.multi_insert(@list, :commit_every => 2)
     @db.sqls.should == [
       'BEGIN',
@@ -2065,6 +2072,19 @@ context "Dataset#multi_insert" do
       'COMMIT'
     ]
   end
+
+  specify "should accept the :slice option for committing every x records" do
+    @ds.multi_insert(@list, :slice => 2)
+    @db.sqls.should == [
+      'BEGIN',
+      "INSERT INTO items (name) VALUES ('abc')",
+      "INSERT INTO items (name) VALUES ('def')",
+      'COMMIT',
+      'BEGIN',
+      "INSERT INTO items (name) VALUES ('ghi')",
+      'COMMIT'
+    ]
+  end
 end
 
 context "Dataset#query" do
@@ -2310,39 +2330,59 @@ context "Dataset#transform" do
     f.should == {:x => "wow", :y => 'hello'}
   end
   
-  specify "should support stock Marshal transformation" do
+  specify "should support stock Marshal transformation with Base64 encoding" do
     @ds.transform(:x => :marshal)
 
-    @ds.raw = {:x => Marshal.dump([1, 2, 3]), :y => 'hello'}
+    @ds.raw = {:x => Base64.encode64(Marshal.dump([1, 2, 3])), :y => 'hello'}
     @ds.first.should == {:x => [1, 2, 3], :y => 'hello'}
 
     @ds.insert(:x => :toast)
-    @ds.sql.should == "INSERT INTO items (x) VALUES ('#{Marshal.dump(:toast)}')"
+    @ds.sql.should == "INSERT INTO items (x) VALUES ('#{Base64.encode64(Marshal.dump(:toast))}')"
     @ds.insert(:y => 'butter')
     @ds.sql.should == "INSERT INTO items (y) VALUES ('butter')"
     @ds.update(:x => ['dream'])
-    @ds.sql.should == "UPDATE items SET x = '#{Marshal.dump(['dream'])}'"
+    @ds.sql.should == "UPDATE items SET x = '#{Base64.encode64(Marshal.dump(['dream']))}'"
 
     @ds2 = @ds.filter(:a => 1)
-    @ds2.raw = {:x => Marshal.dump([1, 2, 3]), :y => 'hello'}
+    @ds2.raw = {:x => Base64.encode64(Marshal.dump([1, 2, 3])), :y => 'hello'}
     @ds2.first.should == {:x => [1, 2, 3], :y => 'hello'}
     @ds2.insert(:x => :toast)
-    @ds2.sql.should == "INSERT INTO items (x) VALUES ('#{Marshal.dump(:toast)}')"
+    @ds2.sql.should == "INSERT INTO items (x) VALUES ('#{Base64.encode64(Marshal.dump(:toast))}')"
 
     @ds.set_row_proc {|r| r[:z] = r[:x] * 2; r}
-    @ds.raw = {:x => Marshal.dump("wow"), :y => 'hello'}
+    @ds.raw = {:x => Base64.encode64(Marshal.dump("wow")), :y => 'hello'}
     @ds.first.should == {:x => "wow", :y => 'hello', :z => "wowwow"}
     f = nil
-    @ds.raw = {:x => Marshal.dump("wow"), :y => 'hello'}
+    @ds.raw = {:x => Base64.encode64(Marshal.dump("wow")), :y => 'hello'}
     @ds.each(:naked => true) {|r| f = r}
     f.should == {:x => "wow", :y => 'hello'}
   end
   
+  specify "should support loading of Marshalled values without Base64 encoding" do
+    @ds.transform(:x => :marshal)
+
+    @ds.raw = {:x => Marshal.dump([1,2,3]), :y => nil}
+    @ds.first.should == {:x => [1,2,3], :y => nil}
+  end
+  
   specify "should return self" do
     @ds.transform(:x => :marshal).should be(@ds)
   end
 end
 
+context "A dataset with a transform" do
+  setup do
+    @ds = Sequel::Dataset.new(nil).from(:items)
+    @ds.transform(:x => :marshal)
+  end
+  
+  specify "should automatically transform hash filters" do
+    @ds.filter(:y => 2).sql.should == 'SELECT * FROM items WHERE (y = 2)'
+    
+    @ds.filter(:x => 2).sql.should == "SELECT * FROM items WHERE (x = '#{Base64.encode64(Marshal.dump(2))}')"
+  end
+end
+
 context "Dataset#to_csv" do
   setup do
     @c = Class.new(Sequel::Dataset) do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: sequel_core
 version: !ruby/object:Gem::Version 
-  version: 1.0.9.1
+  version: 1.0.10
 platform: ruby
 authors: 
 - Sharon Rosner
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-02-11 00:00:00 +02:00
+date: 2008-02-14 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency