mysql2xxxx 0.0.3 → 0.0.4

data/benchmark/results/0.0.4-20110315110535-ruby18.txt

@@ -0,0 +1,149 @@
+mysql2xxxx
+Version: 0.0.4
+Run:     Tue Mar 15 11:05:35 -0500 2011
+System:  Darwin vidalia 10.6.0 Darwin Kernel Version 10.6.0: Wed Nov 10 18:13:17 PST 2010; root:xnu-1504.9.26~3/RELEASE_I386 i386
+
+##################################################
+# CSV
+##################################################
+Real memory over time (sampled every 0.3 sec):
+  0:00.30  11672
+  0:00.60  19936
+  0:00.91  19408
+  0:01.29  20460
+  0:01.56  20460
+  0:01.87  20460
+  0:02.17  20460
+Memprof object counts (top 25):
+   5044 ./benchmark/../lib/mysql2xxxx/writer.rb:43:String
+   5019 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1798:String
+   1617 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1852:String
+    808 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1849:String
+    408 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1849:Array
+    406 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1092:__scope__
+    405 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1091:Hash
+    404 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1852:Method
+    404 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1852:MatchData
+    404 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1490:String
+    372 (__FORWARDABLE__):5:__node__
+    279 (__FORWARDABLE__):3:__node__
+    217 (__FORWARDABLE__):1:__node__
+    206 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1092:String
+    205 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1826:String
+    205 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1734:String
+    205 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1423:Hash
+    204 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:2004:Hash
+    204 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1831:Hash
+    204 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1830:Regexp
+    204 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1826:Regexp
+    204 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1823:String
+    204 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1823:Regexp
+    204 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1820:Regexp
+    204 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/fastercsv-1.5.4/lib/faster_csv.rb:1791:Proc
+
+##################################################
+# JSON
+##################################################
+Real memory over time (sampled every 0.3 sec):
+  0:00.30  12364
+  0:00.61  17128
+  0:00.94  17536
+  0:01.24  17536
+  0:01.54  17536
+  0:01.83  17536
+  0:02.12  17536
+  0:02.41  17536
+  0:02.70  17536
+  0:02.93  17536
+  0:03.06  17536
+  0:03.37  17536
+  0:03.68  17536
+  0:03.98  17536
+  0:04.28  17536
+  0:04.58  17536
+  0:04.88  17536
+  0:05.18  17536
+  0:05.48  17536
+  0:05.79  17536
+  0:06.10  17536
+  0:06.40  17536
+  0:06.71  17536
+  0:07.02  17536
+  0:07.36  17536
+  0:07.67  17536
+  0:07.97  17536
+  0:08.28  17536
+  0:08.59  17536
+  0:08.89  17536
+  0:09.20  17536
+  0:09.52  17536
+  0:09.82  17536
+  0:10.13  17536
+  0:10.44  17536
+  0:10.74  17536
+  0:11.04  17536
+Memprof object counts (top 25):
+   3792 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/activesupport-3.0.5/lib/active_support/json/encoding.rb:125:String
+   2882 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/activesupport-3.0.5/lib/active_support/json/encoding.rb:73:__node__
+   1922 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/activesupport-3.0.5/lib/active_support/json/encoding.rb:235:Array
+   1898 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/activesupport-3.0.5/lib/active_support/json/encoding.rb:132:String
+   1094 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/activesupport-3.0.5/lib/active_support/json/encoding.rb:247:String
+   1022 ./benchmark/../lib/mysql2xxxx/writer.rb:51:String
+    989 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/activesupport-3.0.5/lib/active_support/json/encoding.rb:247:Array
+    950 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/activesupport-3.0.5/lib/active_support/json/encoding.rb:236:Array
+    280 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/yaml/tag.rb:69:__node__
+    260 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/yaml/tag.rb:70:__node__
+    170 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/i686-darwin10.6.0/syck.bundle:0:__node__
+    160 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/yaml/tag.rb:67:__node__
+    160 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/yaml/tag.rb:65:__node__
+    148 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/yaml/tag.rb:75:__node__
+    120 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/i686-darwin10.6.0/stringio.bundle:0:__node__
+    117 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/i686-darwin10.6.0/bigdecimal.bundle:0:__node__
+    114 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/activesupport-3.0.5/lib/active_support/json/encoding.rb:61:Hash
+    105 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/yaml/tag.rb:68:__node__
+     73 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/activesupport-3.0.5/lib/active_support/json/encoding.rb:247:__node__
+     64 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/activesupport-3.0.5/lib/active_support/deprecation/proxy_wrappers.rb:13:__node__
+     63 /Users/seamus/.rvm/gems/ruby-1.8.7-p334/gems/activesupport-3.0.5/lib/active_support/json/encoding.rb:237:__node__
+     60 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/yaml/tag.rb:72:__node__
+     60 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/yaml/tag.rb:71:__node__
+     60 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/yaml/tag.rb:64:__node__
+     60 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/yaml/tag.rb:63:__node__
+
+##################################################
+# XML
+##################################################
+Real memory over time (sampled every 0.3 sec):
+  0:00.29  10920
+  0:00.54  18088
+  0:00.83  17836
+  0:01.12  17496
+  0:01.43  17836
+  0:01.73  18520
+  0:02.03  17496
+  0:02.33  18520
+Memprof object counts (top 25):
+   7638 ./benchmark/../lib/mysql2xxxx/writer/xml.rb:16:String
+   2539 ./benchmark/../lib/mysql2xxxx/writer.rb:43:String
+   2538 ./benchmark/../lib/mysql2xxxx/writer.rb:7:Array
+   2537 ./benchmark/../lib/mysql2xxxx/writer/xml.rb:16:Array
+    120 /Users/seamus/.rvm/rubies/ruby-1.8.7-p334/lib/ruby/1.8/i686-darwin10.6.0/stringio.bundle:0:__node__
+    109 ./benchmark/../lib/mysql2xxxx/writer/xml.rb:15:__node__
+    103 ./benchmark/../lib/mysql2xxxx/writer/xml.rb:18:String
+    103 ./benchmark/../lib/mysql2xxxx/writer/xml.rb:14:String
+    102 ./benchmark/../lib/mysql2xxxx/writer.rb:43:Array
+     28 ./benchmark/../lib/mysql2xxxx/writer/xml.rb:16:__node__
+     25 ./benchmark/../lib/mysql2xxxx/writer/xml.rb:4:String
+     25 ./benchmark/../lib/mysql2xxxx/writer.rb:11:String
+     22 ./benchmark/../lib/mysql2xxxx/writer.rb:35:__node__
+     20 ./benchmark/../lib/mysql2xxxx/writer.rb:16:__node__
+     14 ./benchmark/../lib/mysql2xxxx/writer/xml.rb:12:__node__
+     14 ./benchmark/../lib/mysql2xxxx/properties.rb:18:__node__
+     12 ./benchmark/../lib/mysql2xxxx/writer.rb:11:__node__
+     12 ./benchmark/../lib/mysql2xxxx/properties.rb:36:__node__
+     12 ./benchmark/../lib/mysql2xxxx/properties.rb:32:__node__
+     12 ./benchmark/../lib/mysql2xxxx/properties.rb:14:__node__
+     12 ./benchmark/../lib/mysql2xxxx/properties.rb:10:__node__
+     11 ./benchmark/../lib/mysql2xxxx/writer/xml.rb:4:__node__
+     10 ./benchmark/../lib/mysql2xxxx/writer.rb:34:__node__
+     10 ./benchmark/../lib/mysql2xxxx/properties.rb:40:__node__
+      9 ./benchmark/../lib/mysql2xxxx/writer.rb:26:__node__

data/benchmark/results/0.0.4-20110315110613-ruby19.txt

@@ -0,0 +1,43 @@
+mysql2xxxx
+Version: 0.0.4
+Run:     2011-03-15 11:06:13 -0500
+System:  Darwin vidalia 10.6.0 Darwin Kernel Version 10.6.0: Wed Nov 10 18:13:17 PST 2010; root:xnu-1504.9.26~3/RELEASE_I386 i386
+
+##################################################
+# CSV
+##################################################
+Real memory over time (sampled every 0.3 sec):
+  0:00.31  12416
+  0:00.61  15264
+  0:00.92  15800
+  0:01.23  15800
+  0:01.54  15800
+Memprof object counts not available (ruby 1.9)
+
+##################################################
+# JSON
+##################################################
+Real memory over time (sampled every 0.3 sec):
+  0:00.31  12424
+  0:00.61  15576
+  0:00.91  15684
+  0:01.22  15852
+  0:01.53  15852
+  0:01.83  15852
+  0:02.14  15852
+  0:02.45  15852
+  0:02.75  15852
+  0:03.06  15852
+  0:03.37  15852
+  0:03.67  15852
+  0:03.97  15852
+Memprof object counts not available (ruby 1.9)
+
+##################################################
+# XML
+##################################################
+Real memory over time (sampled every 0.3 sec):
+  0:00.30  12928
+  0:00.60  14616
+  0:00.91  14616
+Memprof object counts not available (ruby 1.9)

data/benchmark/target.rb

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+unless RUBY_VERSION >= '1.9'
+  require 'rubygems'
+  require 'memprof'
+end
+$LOAD_PATH.unshift File.join(ENV['HERE'], '..', 'lib')
+require 'mysql2xxxx'
+
+format = ARGV[0]
+
+if RUBY_VERSION >= '1.9'
+  # unfortunately can't use memprof
+  a = Mysql2xxxx.const_get(format).new :user => ENV['DBUSER'], :password => ENV['DBPASSWORD'], :database => ENV['DBNAME'], :execute => ENV['EXECUTE']
+  a.to_path ENV['DUMP_PATH']
+else
+  Memprof.track(ENV['MEMPROF_REPORT_PATH']) do
+    a = Mysql2xxxx.const_get(format).new :user => ENV['DBUSER'], :password => ENV['DBPASSWORD'], :database => ENV['DBNAME'], :execute => ENV['EXECUTE']
+    a.to_path ENV['DUMP_PATH']
+  end
+end

data/lib/mysql2xxxx.rb

@@ -1,3 +1,6 @@
+unless RUBY_VERSION >= '1.9'
+  $KCODE = 'U'
+end
 require 'active_support'
 require 'active_support/version'
 %w{
@@ -7,13 +10,13 @@ require 'active_support/version'
 }.each do |active_support_3_requirement|
   require active_support_3_requirement
 end if ::ActiveSupport::VERSION::MAJOR == 3
-require 'mysql2'
+require 'mysql'
 
 module Mysql2xxxx
-  autoload :JSON, 'mysql2xxxx/json'
-  autoload :CSV, 'mysql2xxxx/csv'
-  autoload :XML, 'mysql2xxxx/xml'
+  autoload :JSON, 'mysql2xxxx/writer/json'
+  autoload :CSV, 'mysql2xxxx/writer/csv'
+  autoload :XML, 'mysql2xxxx/writer/xml'
+  autoload :Writer, 'mysql2xxxx/writer'
   autoload :Properties, 'mysql2xxxx/properties'
-  autoload :ExtraOutputs, 'mysql2xxxx/extra_outputs'
   autoload :Cli, 'mysql2xxxx/cli'
 end

data/lib/mysql2xxxx/properties.rb

@@ -6,16 +6,6 @@ module Mysql2xxxx
       @options.stringify_keys!
     end
     
-    def database_config
-      {
-        :username => user,
-        :password => password,
-        :host => host,
-        :port => port,
-        :database => database
-      }
-    end
-
     def user
       options['user'] || active_record_config.try(:[], :username)
     end
@@ -25,13 +15,27 @@ module Mysql2xxxx
     end
     
     def host
-      options['host'] || active_record_config.try(:[], :host)
+      options['host'] || active_record_config.try(:[], :host) || '127.0.0.1'
+    end
+    
+    # Force utf8 as mysql connection charset
+    def charset
+      'utf8'
+    end
+    
+    # Force UTF-8 as ruby string encoding (ruby 1.9 only)
+    def encoding
+      'UTF-8'
     end
     
     def port
       options['port'] || active_record_config.try(:[], :port)
     end
     
+    def socket
+      options['socket'] || active_record_config.try(:[], :socket)
+    end
+    
     def database
       options['database'] || active_record_connection.try(:current_database)
     end

data/lib/mysql2xxxx/version.rb

@@ -1,3 +1,3 @@
 module Mysql2xxxx
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/lib/mysql2xxxx/writer.rb

@@ -0,0 +1,82 @@
+require 'stringio'
+module Mysql2xxxx
+  class Writer
+    attr_reader :properties
+    
+    def initialize(options = {})
+      @properties = Properties.new options
+    end
+    
+    def keys
+      @keys ||= result.fetch_fields.map { |field| field.name }
+    end
+        
+    def last_statement
+      return @last_statement if @last_statement.is_a? ::String
+      statements = properties.execute.split(';').select { |statement| statement.to_s.strip.length > 0 }
+      @last_statement = statements.pop
+      statements.each do |statement|
+        dbh.query statement
+        # but we're not using the results
+      end
+      @last_statement
+    end
+    
+    def result
+      return @result if @result.is_a? ::Mysql::Result
+      dbh.query last_statement
+      @result = dbh.use_result
+    end
+    
+    def dbh
+      return @dbh if @dbh.is_a? ::Mysql
+      @dbh = ::Mysql.init
+      @dbh.options ::Mysql::SET_CHARSET_NAME, properties.charset
+      @dbh.real_connect properties.host, properties.user, properties.password, properties.database, properties.port, properties.socket
+      # so that we can use_result instead of store_result
+      @dbh.query_with_result = false
+      @dbh
+    end
+
+    def stream_arrays
+      raise "dead connection" if @dead
+      while ary = result.fetch_row do
+        yield ary
+      end
+      close
+    end
+    
+    def stream_hashes
+      raise "dead connection" if @dead
+      while hsh = result.fetch_hash do
+        yield hsh
+      end
+      close
+    end
+    
+    def close
+      result.free
+      dbh.close
+      @dead = true
+    end
+    
+    def to_s
+      s = ::StringIO.new
+      to_file s
+      s.rewind
+      s.read
+    end
+    
+    def to_stdout
+      to_file $stdout
+      nil
+    end
+    
+    def to_path(path)
+      f = ::File.open(path, 'w')
+      to_file f
+      f.close
+      nil
+    end
+  end
+end

data/lib/mysql2xxxx/writer/csv.rb

@@ -0,0 +1,17 @@
+if RUBY_VERSION >= '1.9'
+  require 'csv'
+else
+  require 'fastercsv'
+end
+
+module Mysql2xxxx
+  class CSV < Writer
+    def to_file(f)
+      f.write keys.to_csv
+      stream_arrays do |ary|
+        f.write ary.to_csv
+      end
+      nil
+    end
+  end
+end

data/lib/mysql2xxxx/writer/json.rb

@@ -0,0 +1,29 @@
+%w{
+  active_support/json/encoding
+}.each do |active_support_3_requirement|
+  require active_support_3_requirement
+end if ::ActiveSupport::VERSION::MAJOR == 3
+
+module Mysql2xxxx
+  class JSON < Writer
+    def to_file(f)
+      first = true
+      f.write '['
+      stream_hashes do |hsh|
+        if first
+          first = false
+        else
+          f.write ','
+        end
+        if RUBY_VERSION >= '1.9'
+          # the mysql gem isn't encoding aware, so even if mysql returns proper utf-8, ruby 1.9 treats it as ASCII-8BIT
+          # here we force ruby to treat these strings as utf-8 without any further conversion
+          hsh.each { |k, v| hsh[k] = v.nil? ? nil : v.force_encoding(properties.encoding) }
+        end
+        f.write hsh.to_json
+      end
+      f.write ']'
+      nil
+    end
+  end
+end

data/lib/mysql2xxxx/{xml.rb → writer/xml.rb}

@@ -1,30 +1,24 @@
 module Mysql2xxxx
-  class XML
-    include ExtraOutputs
-    
-    attr_reader :properties
-    
-    def initialize(options = {})
-      @properties = Properties.new options
+  class XML < Writer
+    def escaped_keys
+      @escaped_keys ||= keys.map { |k| k.to_xs }
     end
     
+    # sabshere 2/10/11 dkastner pointed out that this doesn't include metadata like later versions of mysql do
     # this matches the xml generated by "mysql --xml"
     # i tried to use builder, but the String#to_xs nonsense got in the way
     def to_file(f)
-      @client = ::Mysql2::Client.new properties.database_config
       f.write %{<?xml version="1.0" encoding="utf-8" ?>}
       f.write %{<resultset statement="#{properties.execute.to_xs}" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">}
-      @client.query(properties.execute).each do |hsh|
+      stream_arrays do |ary|
         f.write %{<row>}
-        hsh.each do |k, v|
-          f.write %{<field name="#{k.to_xs}"#{' xsi:nil="true"' if v.nil?}>#{v.to_s.to_xs}</field>}
+        ary.each_with_index do |v, i|
+          f.write %{<field name="#{escaped_keys[i]}"#{' xsi:nil="true"' if v.nil?}>#{v.to_s.to_xs}</field>}
         end
         f.write %{</row>}
       end
       f.write %{</resultset>}
       nil
-    ensure
-      @client.try :close
     end
   end
 end