flydata 0.5.16 → 0.5.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 145b738aa44e8337c13b426d551c12d62d3f688b
-  data.tar.gz: 9597729174fa3a8d1e55eed087eb77ebd3e39cfd
+  metadata.gz: db5ab3ad859c85fb9fcdb2171ad04b0f2e468eb9
+  data.tar.gz: 1225c311e10046702a2cf89b5b2d3f8114022ee9
 SHA512:
-  metadata.gz: 62a90057efd34adb6be5187d932d53d961f80f4e3f6138baa8dba840aaa0e91e22be96ba58fb5ce117e89d74048a4eb234244e59dde2fbe1ff4d86be3f246ffa
-  data.tar.gz: b0eb15b12279ea4865ef9bd93835bbddbb25a4b7d6b04d1711d14a470e6265224e7ded1c733866c9ea93fc1846e6327d5b116181611170ed54da151685eb59bf
+  metadata.gz: c4c2d116d3279e3794b510898bf385aa7aaa22703afdca15bfcae4d7574764ccaa84731b159a98cac7b745de1f46b6fc4e1cf547e06183ee8e46a2c1abc95b60
+  data.tar.gz: 746ccd2cc7cac24b1745e9c08a3184acaa9b7f37a0ed25f2015dd077acb997a3cf25caa812d17c0f78a5591d95b7dd618a38efb61e3f9722e7da13183df34825

data/VERSION

@@ -1 +1 @@
-0.5.16
+0.5.17

data/ext/flydata/json/json_ext.cpp

@@ -262,7 +262,7 @@ static VALUE rb_json_generate_kv_pairs(VALUE self, VALUE key_ary, VALUE value_ar
   }
   ss << "}";
 
-  std::string str = ss.str();
+  const std::string& str = ss.str();
   VALUE json = rb_str_new(str.data(), str.size());
   int enc = rb_enc_find_index("UTF-8");
   rb_enc_associate_index(json, enc);

data/ext/flydata/parser/mysql/sql_parser.cpp

@@ -54,7 +54,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case value:
-        if ($DEBUG) std::cout << "value[" << *current << "]: " << std::endl << std::flush;
+        if ($DEBUG) std::cout << "value[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         if (*current == '\'') {
           status = first_char;
         } else if (*current == 'N') {
@@ -76,7 +76,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case first_zero:
-        if ($DEBUG) std::cout << "first_zero[" << *current << "]: " << std::endl << std::flush;
+        if ($DEBUG) std::cout << "first_zero[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         if (*current == ',') {
           ch.value_callback(mark, value_len - 1, true);//value_callback :end_value
           mark = 0; value_len = 0; //mark_reset
@@ -102,7 +102,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case leading_zero:
-        if ($DEBUG) std::cout << "leading_zero[" << *current << "]: " << std::endl << std::flush;
+        if ($DEBUG) std::cout << "leading_zero[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         if (*current == ',') {
           ch.value_callback(mark, value_len - 1, true);//value_callback :end_value
           mark = 0; value_len = 0; //mark_reset
@@ -126,7 +126,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case number:
-           if ($DEBUG) std::cout << "number[" << *current << "]: " << std::endl << std::flush;
+           if ($DEBUG) std::cout << "number[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         if ((*current >= '0' && *current <= '9') || *current == '.' || *current == 'e' || *current == 'E' || *current == '-') {
           status = number;
         } else if (*current == ',') {
@@ -144,7 +144,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case null_n:
-            if ($DEBUG) std::cout << "null_n[" << *current << "]: " << std::endl << std::flush;
+            if ($DEBUG) std::cout << "null_n[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
      if (*current == 'U') {
           status = null_u;
         } else {
@@ -153,7 +153,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case null_u:
-             if ($DEBUG) std::cout << "null_u[" << *current << "]: " << std::endl << std::flush;
+             if ($DEBUG) std::cout << "null_u[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
     if (*current == 'L') {
           status = null_l1;
         } else {
@@ -162,7 +162,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case null_l1:
-        if ($DEBUG) std::cout << "null_l1[" << *current << "]: " << std::endl << std::flush;
+        if ($DEBUG) std::cout << "null_l1[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         if (*current == 'L') {
           status = null_l2;
         } else {
@@ -171,7 +171,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
        break;
 
       case null_l2:
-         if ($DEBUG) std::cout << "null_l2[" << *current << "]: " << std::endl << std::flush;
+         if ($DEBUG) std::cout << "null_l2[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         if (*current == ',') {
           //# 0 is NULL pointer
           ch.value_callback(0, 0, true);//value_callback 0 :end_value
@@ -189,7 +189,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
       
       case hex_blob:
-        if ($DEBUG) std::cout << "hex_blob[" << *current << "]: " << std::endl << std::flush;
+        if ($DEBUG) std::cout << "hex_blob[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         if ((*current >= '0' && *current <= '9') ||
             (*current >= 'A' && *current <= 'F') ||
             (*current >= 'a' && *current <= 'f')) {
@@ -200,7 +200,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
        break;
 
       case hex_blob_number:
-        if ($DEBUG) std::cout << "hex_blob_number[" << *current << "]: " << std::endl << std::flush;
+        if ($DEBUG) std::cout << "hex_blob_number[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         if ((*current >= '0' && *current <= '9') ||
             (*current >= 'A' && *current <= 'F') ||
             (*current >= 'a' && *current <= 'f')) {
@@ -220,7 +220,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case first_char:
-         if ($DEBUG) std::cout << "first_char[" << *current << "]: " << std::endl << std::flush;
+         if ($DEBUG) std::cout << "first_char[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         if (*current == '\'') {
            mark = current; value_len = 1; //mark
            ch.value_callback(mark, value_len - 1, true);//value_callback, :end_value
@@ -248,8 +248,8 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case utf8_2_firstbyte:
-        if ($DEBUG) std::cout << "utf8_2_firstbyte[" << *current << "]: " << std::endl << std::flush;
-        if ((*current & 0xe0) == 0x80) {
+        if ($DEBUG) std::cout << "utf8_2_firstbyte[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
+        if ((*current & 0xc0) == 0x80) {
           temp_mark = 0; //temp_mark reset)
           status = following_char;
         } else {
@@ -262,8 +262,8 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case utf8_3_firstbyte:
-        if ($DEBUG) std::cout << "utf8_3_firstbyte[" << *current << "]: " << std::endl << std::flush;
-        if ((*current & 0xe0) == 0x80) {
+        if ($DEBUG) std::cout << "utf8_3_firstbyte[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
+        if ((*current & 0xc0) == 0x80) {
           status = utf8_3_secondbyte;
         } else {
           status = utf8_3_err_secondbyte;
@@ -271,8 +271,8 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case utf8_3_secondbyte:
-        if ($DEBUG) std::cout << "utf8_3_secondbyte[" << *current << "]: " << std::endl << std::flush;
-        if ((*current & 0xe0) == 0x80) {
+        if ($DEBUG) std::cout << "utf8_3_secondbyte[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
+        if ((*current & 0xc0) == 0x80) {
           temp_mark = 0; //temp_mark reset)
           status = following_char;
         } else {
@@ -285,7 +285,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case utf8_3_err_secondbyte:
-        if ($DEBUG) std::cout << "utf8_3_err_secondbyte[" << *current << "]: " << std::endl << std::flush;
+        if ($DEBUG) std::cout << "utf8_3_err_secondbyte[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         ch.value_callback(mark, temp_mark - mark, false);//value_callback mark...temp_mark
         mark = 0; value_len = 0; //mark_reset
         ch.value_callback("\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd", 9, false); //value_callback "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"
@@ -294,8 +294,8 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case utf8_4_firstbyte:
-        if ($DEBUG) std::cout << "utf8_4_firstbyte[" << *current << "]: " << std::endl << std::flush;
-        if ((*current & 0xe0) == 0x80) {
+        if ($DEBUG) std::cout << "utf8_4_firstbyte[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
+        if ((*current & 0xc0) == 0x80) {
           status = utf8_4_secondbyte;
         } else {
           status = utf8_4_err_secondbyte;
@@ -303,8 +303,8 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case utf8_4_secondbyte:
-        if ($DEBUG) std::cout << "utf8_4_secondbyte[" << *current << "]: " << std::endl << std::flush;
-        if ((*current & 0xe0) == 0x80) {
+        if ($DEBUG) std::cout << "utf8_4_secondbyte[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
+        if ((*current & 0xc0) == 0x80) {
           status = utf8_4_thirdbyte;
         } else {
           status = utf8_4_err_thirdbyte;
@@ -312,13 +312,13 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case utf8_4_err_secondbyte:
-        if ($DEBUG) std::cout << "utf8_4_err_secondbyte[" << *current << "]: " << std::endl << std::flush;
+        if ($DEBUG) std::cout << "utf8_4_err_secondbyte[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         status = utf8_4_err_thirdbyte;
         break;
 
       case utf8_4_thirdbyte:
-        if ($DEBUG) std::cout << "utf8_4_thirdbyte[" << *current << "]: " << std::endl << std::flush;
-        if ((*current & 0xe0) == 0x80) {
+        if ($DEBUG) std::cout << "utf8_4_thirdbyte[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
+        if ((*current & 0xc0) == 0x80) {
           temp_mark = 0; //temp_mark reset)
           status = following_char;
         } else {
@@ -331,7 +331,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case utf8_4_err_thirdbyte:
-        if ($DEBUG) std::cout << "utf8_4_err_thirdbyte[" << *current << "]: " << std::endl << std::flush;
+        if ($DEBUG) std::cout << "utf8_4_err_thirdbyte[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         ch.value_callback(mark, temp_mark - mark, false);//value_callback mark...temp_mark
         mark = 0; value_len = 0; //mark_reset
         ch.value_callback("\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd", 12, false); //value_callback "\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd"
@@ -340,7 +340,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case following_char:
-         if ($DEBUG) std::cout << "following_char[" << *current << "]: " << std::endl << std::flush;
+         if ($DEBUG) std::cout << "following_char[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         if (*current == '\'') {
           ch.value_callback(mark, value_len - 1, true);//value_callback, :end_value
           mark = 0; value_len = 0; //mark_reset
@@ -364,7 +364,7 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case escape:
-        if ($DEBUG) std::cout << "escape[" << *current << "]: " << std::endl << std::flush;
+        if ($DEBUG) std::cout << "escape[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         if (*current == '\"') {
           ch.value_callback("\"", 1, false);//value_callback '\"'
           mark = 0; value_len = 0;
@@ -401,12 +401,12 @@ long parse_insert_query(const char* sql, long len, ParserCallbackHandler& ch)
         break;
 
       case error:
-       if ($DEBUG) std::cout << "error[" << *current << "]: " << std::endl << std::flush;
+       if ($DEBUG) std::cout << "error[" << *current << std::endl << std::flush;
        ret = pos - 1;
        break;
 
       default:
-        if ($DEBUG) std::cout << "default[" << *current << "]: " << std::endl << std::flush;
+        if ($DEBUG) std::cout << "default[" << *current << "(0x" << std::hex << (*current & 0xff) << ")]: " << std::endl << std::flush;
         break;
     }//switch
     if (ret != -1)

data/lib/flydata/command/exclusive_runnable.rb

@@ -21,6 +21,13 @@ module ExclusiveRunnable
         begin
           info = self.class.load_exclusive_run_info
           if info
+            #TODO If the previous command terminates abnormally, 
+            # 'exclusive_run.info' file won't be deleted even the pid 
+            # doesn't exist anymore. 
+            # This prevents execution of any commands afterwards, and 
+            # the user will see this misleading error message. 
+            # Introducing "--force-run" option can be a solution. 
+
             raise "Command `#{info['command']}` is already running.  Wait until the command completes or stop the command.  (pid:#{info['pid']})"
           end
           exclusive_run_info = { command: self.class.command(method, options), pid: Process.pid }

data/lib/flydata/command/sync.rb

@@ -177,6 +177,7 @@ EOS
         if @input_tables.empty? or @full_tables.empty? or @full_tables.all?{|ft| new_tables_after_reset.include?(ft)}
           delete_files << sync_fm.binlog_path
           delete_files << sync_fm.sent_binlog_path
+          delete_files << sync_fm.lock_pid_file
         end
         delete_files.flatten.each do |path|
           FileUtils.rm(path) if File.exists?(path)

data/lib/flydata/sync_file_manager.rb

@@ -240,6 +240,10 @@ module Flydata
       File.open(file) {|f| f.read}
     end
 
+    def lock_pid_file
+      FLYDATA_LOCK
+    end
+
     def sync_info_file
       File.join(dump_dir, "sync.info")
     end

data/spec/flydata/parser/mysql/dump_parser_spec.rb

@@ -291,7 +291,7 @@ CREATE TABLE `users_login` (
 
 LOCK TABLES `users_login` WRITE;
 /*!40000 ALTER TABLE `users_login` DISABLE KEYS */;
-INSERT INTO `users_login` VALUES (15,46,'moodier\xED\xB1\xA2','2001-10-02 08:18:08','1986-06-11 22:50:10'),(35,6,'missあいうえおteer','1991-10-15 19:38:07','1970-10-01 22:03:10'),(52,33,'sub\\\\field','1972-08-23 20:16:08','1974-10-10 23:28:11');
+INSERT INTO `users_login` VALUES (15,46,'moodier\xED\xB1\x22','2001-10-02 08:18:08','1986-06-11 22:50:10'),(35,6,'missあいうえおteer','1991-10-15 19:38:07','1970-10-01 22:03:10'),(52,33,'sub\\\\field','1972-08-23 20:16:08','1974-10-10 23:28:11');
 INSERT INTO `users_login` VALUES (373,31,'out\\'swearing','1979-10-07 08:10:08','2006-02-22 16:26:04'),(493,8,'schizophrenic','1979-07-06 07:34:07','1970-08-09 01:21:01');
 /*!40000 ALTER TABLE `users_login` ENABLE KEYS */;
 UNLOCK TABLES;
@@ -1080,6 +1080,33 @@ EOS
               expect(subject).to eq([["test\x1ctest"]])
             end
           end
+
+          context 'with various characters' do
+            context 'UTF-8 1 byte char' do
+              let(:target_line) {"INSERT INTO `test_table` VALUES ('testtest');"}
+              it 'should convert correctly' do
+                expect(subject).to eq([["testtest"]])
+              end
+            end
+            context 'UTF-8 2 byte char' do
+              let(:target_line) {"INSERT INTO `test_table` VALUES ('testðtest');"}
+              it 'should convert correctly' do
+                expect(subject).to eq([["testðtest"]])
+              end
+            end
+            context 'UTF-8 3 byte char' do
+              let(:target_line) {"INSERT INTO `test_table` VALUES ('testあtest');"}
+              it 'should convert correctly' do
+                expect(subject).to eq([["testあtest"]])
+              end
+            end
+            context 'UTF-8 4 byte char' do
+              let(:target_line) {"INSERT INTO `test_table` VALUES ('test\xf0\x90\x85\x83test');"}
+              it 'should convert correctly' do
+                expect(subject).to eq([["test\xf0\x90\x85\x83test"]])
+              end
+            end
+          end
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.16
+  version: 0.5.17
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-02 00:00:00.000000000 Z
+date: 2015-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client