flydata 0.6.12 → 0.6.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 181cbcfe32036e4e5e580f5208c183c2aeede5d5
-  data.tar.gz: d74e286636bafd5a3ade8482cd808c69597ff723
+  metadata.gz: 4abef5efd4f51c0f72fc28e582eb3331709e31d3
+  data.tar.gz: 2b457b9f88d81ea0e16fc810c1bc5a4e6a7dbf8a
 SHA512:
-  metadata.gz: d01fd1622e3573ed4b2989af7ad77a1fd9f8c945bc6d8f959e65818a4733c161e32793c908c51ce71974066bc86db80d69aabff904d18a4336ddc60f95bd40bf
-  data.tar.gz: f829d5633875735e6fef04eec4fadd7778b9e35eedbc7decbcbea150ebb4a2bf64d91bfcc50550f14f4a59ff8aecc2599cae93bb249e8ec775d2b58a71927f43
+  metadata.gz: c6cc67ea772223bc4f8d64dadc8a2e76a118ad5de04f6e165a24696f2f3a0eefc1e1a87c052d48877664fc70bb47c7600fbe5ce8baa5f833e805402e2f95822f
+  data.tar.gz: ebaf4bb22696a9b0c8e5013d0792063e7e054134b721e599915562f72372fdd953f61bea596d65e4de6aedd255bcd54c2fd958ced67910979cbfb61c8a801d41

data/Gemfile

@@ -14,7 +14,7 @@ gem "slop", '~> 3.4', '>= 3.4.6'
 gem "treetop", '~> 1.5', '>= 1.5.3'
 gem "sys-filesystem", '~> 1.1', '>= 1.1.3'
 gem "io-console", '~> 0.4.2', '>= 0.4.2'
-gem "kodama", '~> 0.1', '>= 0.1.11'
+gem "kodama", '~> 0.1', '>= 0.1.12'
 gem "serverengine", '~> 1.5'
 
 group :development do

data/Gemfile.lock

@@ -61,8 +61,8 @@ GEM
       rdoc
     json (1.8.1)
     jwt (1.0.0)
-    kodama (0.1.11)
-      ruby-binlog (~> 1.0, >= 1.0.9)
+    kodama (0.1.12)
+      ruby-binlog (~> 1.0, >= 1.0.10)
     method_source (0.8.2)
     mime-types (2.3)
     minitest (4.7.5)
@@ -108,7 +108,7 @@ GEM
     rspec-mocks (3.0.3)
       rspec-support (~> 3.0.0)
     rspec-support (3.0.3)
-    ruby-binlog (1.0.9)
+    ruby-binlog (1.0.10)
     ruby-prof (0.15.1)
     serverengine (1.5.10)
       sigdump (~> 0.2.2)
@@ -138,7 +138,7 @@ DEPENDENCIES
   io-console (~> 0.4.2, >= 0.4.2)
   jeweler (~> 1.8, >= 1.8.8)
   json (~> 1.8, >= 1.8.0)
-  kodama (~> 0.1, >= 0.1.11)
+  kodama (~> 0.1, >= 0.1.12)
   mysql2 (~> 0.3, >= 0.3.17)
   pg (~> 0.18.4)
   protected_attributes (~> 1.0, >= 1.0.8)

data/VERSION

@@ -1 +1 @@
-0.6.12
+0.6.13

data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb

@@ -1,4 +1,5 @@
 require 'flydata-core/table_def/base'
+require 'flydata-core/string_utils'
 
 module FlydataCore
 module TableDef
@@ -101,6 +102,7 @@ class MysqlTableDef < Base
     position = :before_create_table
 
     io.each_line do |line|
+      line = FlydataCore::StringUtils.replace_invalid_utf8_char(line)
       case position
       when :before_create_table
         if line =~ /CREATE TABLE `(.*?)`/

data/flydata-core/spec/table_def/mysql_table_def_spec.rb

@@ -128,6 +128,19 @@ describe MysqlTableDef do
           ]
         )
       end
+
+      context 'when column comment contains includes invalid chars' do
+        let(:dump_file_io) { file_io('mysqldump_test_table_column_comment_including_invalid_char.dump') }
+        it 'comment should be set with replaced chars' do
+          expect(subject[:columns]).to eq(
+            [
+              {:column=>"id", :type=>"int4(11)", :not_null=>true, :default=>"'0'",
+               :comment=>"a?2BY???i", :primary_key=>true},
+               {:column=>"value", :type=>"text", :comment=>"あいうえお"}
+            ]
+          )
+        end
+      end
     end
 
     context 'when table has multiple pk' do

data/flydata-core/spec/table_def/mysqldump_test_table_column_comment_including_invalid_char.dump

@@ -0,0 +1,41 @@
+-- MySQL dump 10.13  Distrib 5.6.13, for osx10.9 (x86_64)
+--
+-- Host: localhost    Database: masashi_test
+-- ------------------------------------------------------
+-- Server version	5.6.13-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `test_table_column_comment`
+--
+
+DROP TABLE IF EXISTS `test_table_column_comment`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `test_table_column_comment` (
+  `id` int(11) NOT NULL DEFAULT '0' COMMENT 'a�2BY���i',
+  `value` text COMMENT 'あいうえお',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='column comment test';
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2014-04-25 13:24:35

data/lib/flydata/command/sync.rb

@@ -280,7 +280,8 @@ EOS
       #  - Entry method
       def check(options = {})
         context = source.source_pos
-        status, pos_mismatch_tables, gap_tables = _check(context, options)
+        status, corrupt_master_pos_files, pos_mismatch_tables, gap_tables =
+          _check(context, options)
 
         if status.include? :OK
           message = "\nNo errors are found.  Sync is clean.\n"
@@ -296,6 +297,9 @@ EOS
           if status.include? :ABNORMAL_SHUTDOWN
             message += " - Agent was not shut down correctly\n"
           end
+          if status.include? :CORRUPT_MASTER_POS
+            message += " - Master source position is corrupted\n"
+          end
           if gap_tables
             message += " - Sync data is missing for the following table(s)\n"
             gap_tables.each do |bt|
@@ -462,6 +466,13 @@ EOS
             next
           end
           if tables.empty?
+            # Find the latest source pos to which all table data is up to date
+            if table_status_hash[table]["seq"] == 0
+              # this table's source pos is from the initial sync.  It can't
+              # be used for determining the latest source pos because it can be
+              # newer than the correct cont sync source pos.
+              next
+            end
             if sent_source_pos.nil? || sent_source_pos < table_source_pos
               sent_source_pos = table_source_pos
             end

data/lib/flydata/source_mysql/parser/dump_parser.rb

@@ -332,6 +332,7 @@ EOS
           state_create_table_columns = Proc.new do
             start_pos = dump_io.pos
             line = readline_proc.call
+            line = FlydataCore::StringUtils.replace_invalid_utf8_char(line)
 
             #  `author_type` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
             if line.start_with?("\`")

data/spec/flydata/source_mysql/parser/dump_parser_spec.rb

@@ -280,7 +280,7 @@ DROP TABLE IF EXISTS `users_login`;
 CREATE TABLE `users_login` (
   `user_id` int(10) unsigned NOT NULL DEFAULT '0' COMMENT 'users id',
   `login_count` int(10) unsigned DEFAULT '0' COMMENT 'login count',
-  `comment` varchar(255) DEFAULT NULL COMMENT 'comment',
+  `comment` varchar(255) DEFAULT NULL COMMENT 'comment\xED\xB1\x22end',
   `create_time` datetime NOT NULL COMMENT 'create time',
   `update_time` datetime NOT NULL COMMENT 'update time',
   PRIMARY KEY (`user_id`)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.6.12
+  version: 0.6.13
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-09 00:00:00.000000000 Z
+date: 2016-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -262,7 +262,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.11
+        version: 0.1.12
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.1'
@@ -272,7 +272,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.11
+        version: 0.1.12
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.1'
@@ -590,6 +590,7 @@ files:
 - flydata-core/spec/table_def/mysqldump_test_foreign_key.dump
 - flydata-core/spec/table_def/mysqldump_test_table_all.dump
 - flydata-core/spec/table_def/mysqldump_test_table_column_comment.dump
+- flydata-core/spec/table_def/mysqldump_test_table_column_comment_including_invalid_char.dump
 - flydata-core/spec/table_def/mysqldump_test_table_enum.dump
 - flydata-core/spec/table_def/mysqldump_test_table_multi_pk.dump
 - flydata-core/spec/table_def/mysqldump_test_table_no_pk.dump