sq-dbsync 1.0.1 → 1.0.2

data/lib/sq/dbsync/database/connection.rb

@@ -12,7 +12,9 @@ module Sq::Dbsync::Database
     def self.create(opts)
       case opts[:brand]
       when 'mysql'
-        Sq::Dbsync::Database::Mysql.new(Sequel.connect(opts))
+        db = Sq::Dbsync::Database::Mysql.new(Sequel.connect(opts))
+        db.charset = opts[:charset] if opts[:charset]
+        db
       when 'postgresql'
         Sq::Dbsync::Database::Postgres.new(Sequel.connect(opts))
       else

data/lib/sq/dbsync/database/mysql.rb

@@ -16,6 +16,8 @@ module Sq::Dbsync::Database
 
     include Common
 
+    attr_accessor :charset
+
     def initialize(db)
       super
       @db = db
@@ -25,9 +27,11 @@ module Sq::Dbsync::Database
 
     def load_from_file(table_name, columns, file_name)
       ensure_connection
-      sql = "LOAD DATA INFILE '%s' IGNORE INTO TABLE %s (%s)" % [
+      character_set = self.charset ? " character set #{self.charset}" : ""
+      sql = "LOAD DATA INFILE '%s' IGNORE INTO TABLE %s %s (%s)" % [
         file_name,
         table_name,
+        character_set,
         escape_columns(columns)
       ]
       db.run sql
@@ -42,9 +46,11 @@ module Sq::Dbsync::Database
       # Very low lock wait timeout, since we don't want loads to be blocked
       # waiting for long queries.
       set_lock_timeout(10)
-      db.run "LOAD DATA INFILE '%s' REPLACE INTO TABLE %s (%s)" % [
+      character_set = self.charset ? " character set #{self.charset}" : ""
+      db.run "LOAD DATA INFILE '%s' REPLACE INTO TABLE %s %s (%s)" % [
         file_name,
         table_name,
+        character_set,
         escape_columns(columns)
       ]
     rescue Sequel::DatabaseError => e
@@ -134,6 +140,9 @@ module Sq::Dbsync::Database
       cmd += " -p%s"    % opts[:password] if opts[:password]
       cmd += " -h %s"   % opts[:host]     if opts[:host]
       cmd += " -P %i"   % opts[:port]     if opts[:port]
+
+      cmd += " --default-character-set %s" % opts[:charset] if opts[:charset]
+
       cmd += " %s"      % opts.fetch(:database)
 
       # This option prevents mysql from buffering results in memory before

data/lib/sq/dbsync/version.rb

@@ -1,5 +1,5 @@
 module Sq
   module Dbsync
-    VERSION = '1.0.1'
+    VERSION = '1.0.2'
   end
 end

data/spec/acceptance/loading_spec.rb

@@ -14,13 +14,14 @@ describe 'Syncing source databases to a target' do
     SQD::Manager.new(config, [[SQD::StaticTablePlan.new(plan), :source]])
   }
   let(:source)     { manager.sources.fetch(:source) }
+  let(:mb4_source) { manager.sources.fetch(:mb4_source) }
   let(:alt_source) { manager.sources.fetch(:alt_source) }
   let(:target) { manager.target }
   let(:plan) {[{
     table_name: :test_table,
     source_table_name: :test_table,
     refresh_recent: true,
-    columns:    [:id, :updated_at]
+    columns:    [:id, :updated_at, :value]
   }] }
 
   MINUTE = 60
@@ -74,6 +75,34 @@ describe 'Syncing source databases to a target' do
       target[:test_table     ].map {|x| x[:id] }.should include(row)
       target[:alt_test_table ].map {|x| x[:id] }.should include(alt_row)
     end
+
+    it 'handles utf8mb4 inputs' do
+      plan = [{
+                table_name: :mb4_test_table,
+                source_table_name: :mb4_test_table,
+                refresh_recent: true,
+                columns: [:id, :updated_at, :value],
+                charset: "utf8mb4"
+              }]
+
+      config = {
+        sources: TEST_SOURCES,
+        target:  MB4_TEST_TARGET,
+        logger:  SQD::Loggers::Composite.new([logger]),
+        clock:   ->{ @now }
+      }
+
+      manager = SQD::Manager.new(config, [[SQD::StaticTablePlan.new(plan), :mb4_source]])
+
+      row = mb4_source[:mb4_test_table].insert(updated_at: @now, value: "\u{1f4a9}")
+
+      manager.batch_nonactive
+
+      manager.target[:mb4_test_table].map {|x| x[:id] }.should include(row)
+      manager.target[:mb4_test_table].map {|x| x[:value] }.should include("\u{1f4a9}")
+      manager.target[:meta_last_sync_times].count.should == 1
+    end
+
   end
 
   context 'refresh recent load' do
@@ -211,6 +240,13 @@ describe 'Syncing source databases to a target' do
       primary_key  :id
       DateTime :reporting_date
       DateTime :updated_at
+      String :value
+    end
+    mb4_source.create_table! :mb4_test_table, charset:"utf8mb4" do
+      primary_key  :id
+      DateTime :reporting_date
+      DateTime :updated_at
+      String :value
     end
     source.create_table! :test_table_2 do
       primary_key  :id

data/spec/database_helper.rb

@@ -34,6 +34,7 @@ end
 
 TEST_SOURCES = {
   source:     db_options(database: 'sq_dbsync_test_source'),
+  mb4_source: db_options(database: 'sq_dbsync_mb4_test_source', charset: "utf8mb4"),
   alt_source: db_options(database: 'sq_dbsync_test_source_alt'),
   postgres:   db_options(
     user:     `whoami`.chomp,
@@ -43,6 +44,7 @@ TEST_SOURCES = {
   )
 }
 TEST_TARGET = db_options(database: 'sq_dbsync_test_target')
+MB4_TEST_TARGET = db_options(database: 'sq_dbsync_test_target', charset:"utf8mb4")
 
 $target = nil
 def test_target

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sq-dbsync
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-27 00:00:00.000000000 Z
+date: 2013-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -163,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Column based, timestamp replication of MySQL and Postgres databases. Uses