mysql2xxxx 0.0.4 → 0.1.0

data/README.rdoc

@@ -2,9 +2,9 @@
 
 ==Binaries
 
-    mysql2csv --user=dbuser --password=dbpassword --database=dbname --execute="select * from automobile_makes"
-    mysql2json --user=dbuser --password=dbpassword --database=dbname --execute="select * from automobile_makes"
-    mysql2xml --user=dbuser --password=dbpassword --database=dbname --execute="select * from automobile_makes"
+    mysql2csv --user=dbuser --password=dbpassword --database=dbname [--charset=utf8] [--encoding=UTF-8] --execute="select * from automobile_makes"
+    mysql2json --user=dbuser --password=dbpassword --database=dbname [--charset=utf8] [--encoding=UTF-8] --execute="select * from automobile_makes"
+    mysql2xml --user=dbuser --password=dbpassword --database=dbname [--charset=utf8] [--encoding=UTF-8] --execute="select * from automobile_makes"
 
 To see all options
 

data/lib/mysql2xxxx/cli.rb

@@ -3,7 +3,7 @@ require 'mixlib/cli'
 module Mysql2xxxx
   class Cli
     include ::Mixlib::CLI
-    %w{ user password host Port database }.each do |o|
+    %w{ user password host Port database charset encoding }.each do |o|
       option o.downcase.to_sym,
         :short => "-#{o[0,1]} #{o.upcase}",
         :long  => "--#{o.downcase}=#{o.upcase}",

data/lib/mysql2xxxx/properties.rb

@@ -18,14 +18,22 @@ module Mysql2xxxx
       options['host'] || active_record_config.try(:[], :host) || '127.0.0.1'
     end
     
-    # Force utf8 as mysql connection charset
+    # MySQL connection charset
+    #
+    # If you change this, you also have to change :encoding
+    #
+    # Default: utf8
     def charset
-      'utf8'
+      options['charset'] || 'utf8'
     end
     
-    # Force UTF-8 as ruby string encoding (ruby 1.9 only)
+    # Encoding
+    #
+    # If you change this, you also have to change :charset
+    #
+    # Default: UTF-8
     def encoding
-      'UTF-8'
+      options['encoding'] || 'UTF-8'
     end
     
     def port

data/lib/mysql2xxxx/version.rb

@@ -1,3 +1,3 @@
 module Mysql2xxxx
-  VERSION = "0.0.4"
+  VERSION = "0.1.0"
 end

data/lib/mysql2xxxx/writer.rb

@@ -1,4 +1,10 @@
 require 'stringio'
+if ::RUBY_VERSION >= '1.9'
+  require 'ensure/encoding'
+else
+  require 'iconv'
+end
+
 module Mysql2xxxx
   class Writer
     attr_reader :properties
@@ -41,7 +47,7 @@ module Mysql2xxxx
     def stream_arrays
       raise "dead connection" if @dead
       while ary = result.fetch_row do
-        yield ary
+        yield ary.map { |v| recode_as_utf8 v }
       end
       close
     end
@@ -49,11 +55,26 @@ module Mysql2xxxx
     def stream_hashes
       raise "dead connection" if @dead
       while hsh = result.fetch_hash do
-        yield hsh
+        yield hsh.inject({}) { |memo, (k, v)| memo[recode_as_utf8(k)] = recode_as_utf8(v); memo }
       end
       close
     end
     
+    def recode_as_utf8(raw_str)
+      return if raw_str.nil?
+      if ::RUBY_VERSION >= '1.9'
+        $stderr.puts "[mysql2xxxx] Raw - #{raw_str}" if ::ENV['MYSQL2XXXX_DEBUG'] == 'true'
+        recoded_str = raw_str.ensure_encoding 'UTF-8', :external_encoding => properties.encoding, :invalid_characters => :transcode
+        $stderr.puts "[mysql2xxxx] Recoded - #{recoded_str}" if ::ENV['MYSQL2XXXX_DEBUG'] == 'true'
+        recoded_str
+      else
+        $stderr.puts "[mysql2xxxx] Raw - #{raw_str}" if ::ENV['MYSQL2XXXX_DEBUG'] == 'true'
+        recoded_str = ::Iconv.conv('UTF-8//TRANSLIT', properties.encoding, raw_str.to_s + ' ')[0..-2]
+        $stderr.puts "[mysql2xxxx] Recoded - #{recoded_str}" if ::ENV['MYSQL2XXXX_DEBUG'] == 'true'
+        recoded_str
+      end
+    end
+    
     def close
       result.free
       dbh.close

data/lib/mysql2xxxx/writer/json.rb

@@ -15,11 +15,6 @@ module Mysql2xxxx
         else
           f.write ','
         end
-        if RUBY_VERSION >= '1.9'
-          # the mysql gem isn't encoding aware, so even if mysql returns proper utf-8, ruby 1.9 treats it as ASCII-8BIT
-          # here we force ruby to treat these strings as utf-8 without any further conversion
-          hsh.each { |k, v| hsh[k] = v.nil? ? nil : v.force_encoding(properties.encoding) }
-        end
         f.write hsh.to_json
       end
       f.write ']'

data/mysql2xxxx.gemspec

@@ -23,9 +23,8 @@ Gem::Specification.new do |s|
   s.add_dependency 'mysql'
   s.add_dependency 'mixlib-cli'
   s.add_dependency 'fast_xs'
-  unless RUBY_VERSION >= '1.9'
-    s.add_dependency 'fastercsv'
-  end
+  s.add_dependency 'fastercsv'
+  s.add_dependency 'ensure-encoding'
   
   s.add_development_dependency 'activerecord'
   s.add_development_dependency 'shell-executer'

data/test/test_mysql2xxxx.rb

@@ -6,7 +6,7 @@ class TestMysql2xxxx < Test::Unit::TestCase
     a = Mysql2xxxx::JSON.new @options
     str = a.to_s
     assert str.include?('Acura')
-    assert str.include?('Citro\\u00ebn')
+    assert str.include?(%q{Citro\u00ebn})
     assert !str.include?('DaimlerChrysler')
   end
 

metadata

@@ -2,7 +2,7 @@
 name: mysql2xxxx
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-15 00:00:00 -05:00
+date: 2011-05-08 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -58,7 +58,7 @@ dependencies:
   type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: activerecord
+  name: fastercsv
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
@@ -66,10 +66,10 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version 
         version: "0"
-  type: :development
+  type: :runtime
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
-  name: shell-executer
+  name: ensure-encoding
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
@@ -77,10 +77,10 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version 
         version: "0"
-  type: :development
+  type: :runtime
   version_requirements: *id006
 - !ruby/object:Gem::Dependency 
-  name: posix-spawn
+  name: activerecord
   prerelease: false
   requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
@@ -91,7 +91,7 @@ dependencies:
   type: :development
   version_requirements: *id007
 - !ruby/object:Gem::Dependency 
-  name: ruby-debug19
+  name: shell-executer
   prerelease: false
   requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
@@ -101,6 +101,28 @@ dependencies:
         version: "0"
   type: :development
   version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: posix-spawn
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: ruby-debug19
+  prerelease: false
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id010
 description: Gives you binaries like mysql2csv, mysql2json, and mysql2xml, and Ruby classes to match.
 email: 
 - seamus@abshere.net
@@ -113,6 +135,7 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
+- .gemtest
 - .gitignore
 - Gemfile
 - README.rdoc