flydata 0.5.15 → 0.5.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c4f662e6fc6c4a180d38770b2b84b87292b8d44f
-  data.tar.gz: 97f92499dca46d6d9368c9e81756f502be4951dd
+  metadata.gz: 145b738aa44e8337c13b426d551c12d62d3f688b
+  data.tar.gz: 9597729174fa3a8d1e55eed087eb77ebd3e39cfd
 SHA512:
-  metadata.gz: cf2588a3cce18df232abc2eecd2b066eab5224bb3e0b30d6a80484663f586f5de8c2b821cf22341f5cae7b731bd4d99aed55f6fa9b5ceab5bb06612fa2d5bcb0
-  data.tar.gz: c9b5c2b979ef0360e7d8ce8fd73c8df645559cb56888b5d7a40746700b6e48537fdc5b8c4f453a872a44c2cccd25f440552b86d7a447a633e491faf3cd4a5abc
+  metadata.gz: 62a90057efd34adb6be5187d932d53d961f80f4e3f6138baa8dba840aaa0e91e22be96ba58fb5ce117e89d74048a4eb234244e59dde2fbe1ff4d86be3f246ffa
+  data.tar.gz: b0eb15b12279ea4865ef9bd93835bbddbb25a4b7d6b04d1711d14a470e6265224e7ded1c733866c9ea93fc1846e6327d5b116181611170ed54da151685eb59bf

data/VERSION

@@ -1 +1 @@
-0.5.15
+0.5.16

data/ext/flydata/json/json_ext.cpp

@@ -10,6 +10,16 @@ _declspec(dllexport)
 void Init_json_ext();
 }
 
+inline void write_escape_str(const char*& buf, long& buf_len, const char* escape_str, long escape_str_len, std::stringstream& ss)
+{
+   if (buf_len > 0)
+   {
+     ss.write(buf, buf_len);
+   }
+   ss.write(escape_str, escape_str_len);
+   buf = 0; buf_len = 0;
+}
+
 inline void add_json_value(const char* buf, long buf_len, bool quote, std::stringstream& ss)
 {
   if (quote)
@@ -21,31 +31,162 @@ inline void add_json_value(const char* buf, long buf_len, bool quote, std::strin
   const char* esc_str = 0;
   for (long i = 0; i < buf_len; i++) {
     if (!ptr) { ptr = buf + i; len = 0; }
+
+    if ((buf[i] >= 0x5d && buf[i] < 0x7f) || // small letter alphabet
+        (buf[i] >= 0x30 && buf[i] < 0x5c) || // digit and capital letter alphabet
+        (buf[i] >= 0x80) ||                  // multi-byte utf-8
+        (buf[i] >= 0x23 && buf[i] < 0x2f) || // signs
+        (buf[i] == 0x21)                     // exclamation mark
+       ) {
+      len++;
+      continue;
+    }
+    // Handle JSON special characters.  See http://www.json.org/
     switch (buf[i])
     {
       case '\"':
         esc_str = "\\\"";
+        write_escape_str(ptr, len, esc_str, 2, ss);
+        break;
       case '\\':
         esc_str = "\\\\";
+        write_escape_str(ptr, len, esc_str, 2, ss);
+        break;
       case '/':
         esc_str = "\\/";
+        write_escape_str(ptr, len, esc_str, 2, ss);
+        break;
+      case '\t':
+        esc_str = "\\t";
+        write_escape_str(ptr, len, esc_str, 2, ss);
+        break;
+      case '\n':
+        esc_str = "\\n";
+        write_escape_str(ptr, len, esc_str, 2, ss);
+        break;
+      case 0x00:
+        esc_str = "\\u0000";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x01:
+        esc_str = "\\u0001";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x02:
+        esc_str = "\\u0002";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x03:
+        esc_str = "\\u0003";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x04:
+        esc_str = "\\u0004";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x05:
+        esc_str = "\\u0005";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x06:
+        esc_str = "\\u0006";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case '\a':
+        esc_str = "\\u0007";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
       case '\b':
         esc_str = "\\b";
+        write_escape_str(ptr, len, esc_str, 2, ss);
+        break;
+      case 0x0b:
+        esc_str = "\\u000b";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
       case '\f':
         esc_str = "\\f";
-      case '\n':
-        esc_str = "\\n";
+        write_escape_str(ptr, len, esc_str, 2, ss);
+        break;
       case '\r':
         esc_str = "\\r";
-      case '\t':
-        esc_str = "\\t";
-  
-        if (len > 0)
-        {
-          ss.write(ptr, len);
-        }
-        ss.write(esc_str, 2);
-        ptr = 0; len = 0;
+        write_escape_str(ptr, len, esc_str, 2, ss);
+        break;
+      case 0x0e:
+        esc_str = "\\u000e";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x0f:
+        esc_str = "\\u000f";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x10:
+        esc_str = "\\u0010";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x11:
+        esc_str = "\\u0011";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x12:
+        esc_str = "\\u0012";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x13:
+        esc_str = "\\u0013";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x14:
+        esc_str = "\\u0014";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x15:
+        esc_str = "\\u0015";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x16:
+        esc_str = "\\u0016";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x17:
+        esc_str = "\\u0017";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x18:
+        esc_str = "\\u0018";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x19:
+        esc_str = "\\u0019";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x1a:
+        esc_str = "\\u001a";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x1b:
+        esc_str = "\\u001b";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x1c:
+        esc_str = "\\u001c";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x1d:
+        esc_str = "\\u001d";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x1e:
+        esc_str = "\\u001e";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x1f:
+        esc_str = "\\u001f";
+        write_escape_str(ptr, len, esc_str, 6, ss);
+        break;
+      case 0x7f:
+        esc_str = "\\u007f";
+        write_escape_str(ptr, len, esc_str, 6, ss);
         break;
       default:
         len++;

data/spec/flydata/json/json_ext_spec.rb

@@ -0,0 +1,69 @@
+require 'spec_helper'
+require 'flydata/json/json_ext'
+
+describe JSON do
+  describe '.generate_kv_pairs' do
+    subject { described_class.generate_kv_pairs(keys, values) }
+
+    context 'testing values' do
+      let(:keys) { ['test'] }
+      let(:values) { [ value ] }
+      let(:expected_result) { %Q|{"test":"#{expected_value}"}| }
+      shared_examples 'producing the expected result' do
+        it do
+          is_expected.to eq expected_result
+        end
+      end
+      context 'regular string' do
+        let(:value) { "a regular text" }
+        let(:expected_value) { value }
+        it_behaves_like "producing the expected result"
+      end
+      context 'regular UTF-8 2 byte chars' do
+        let(:value) { "¢Àø" }
+        let(:expected_value) { value }
+        it_behaves_like "producing the expected result"
+      end
+      context 'regular UTF-8 3 byte chars' do
+        let(:value) { "三国一" }
+        let(:expected_value) { value }
+        it_behaves_like "producing the expected result"
+      end
+      context '/' do
+        let(:value) { "/" }
+        let(:expected_value) { "\\/" }
+        it_behaves_like "producing the expected result"
+      end
+      context '\\' do
+        let(:value) { "\\" }
+        let(:expected_value) { "\\\\" }
+        it_behaves_like "producing the expected result"
+      end
+      context '"' do
+        let(:value) { "\"" }
+        let(:expected_value) { "\\\"" }
+        it_behaves_like "producing the expected result"
+      end
+      context '\'' do
+        let(:value) { "'" }
+        let(:expected_value) { "'" }
+        it_behaves_like "producing the expected result"
+      end
+      context '\b' do
+        let(:value) { "\b" }
+        let(:expected_value) { "\\b" }
+        it_behaves_like "producing the expected result"
+      end
+      context '\x02' do
+        let(:value) { "\x02" }
+        let(:expected_value) { "\\u0002" }
+        it_behaves_like "producing the expected result"
+      end
+      context '^\\' do
+        let(:value) { "\x1c" }
+        let(:expected_value) { "\\u001c" }
+        it_behaves_like "producing the expected result"
+      end
+    end
+  end
+end

data/spec/flydata/parser/mysql/dump_parser_spec.rb

@@ -1068,6 +1068,18 @@ EOS
             end
           end
 
+          context 'when ^B is given' do
+            let(:target_line) {"INSERT INTO `test_table` VALUES ('testtest');"}
+            it 'should not remove the leading zeros' do
+              expect(subject).to eq([["test\x02test"]])
+            end
+          end
+          context 'when ^\\ is given' do
+            let(:target_line) {"INSERT INTO `test_table` VALUES ('testtest');"}
+            it 'should not remove the leading zeros' do
+              expect(subject).to eq([["test\x1ctest"]])
+            end
+          end
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.15
+  version: 0.5.16
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-29 00:00:00.000000000 Z
+date: 2015-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -696,6 +696,7 @@ files:
 - spec/flydata/helper/scheduler_spec.rb
 - spec/flydata/helper/worker_spec.rb
 - spec/flydata/heroku_spec.rb
+- spec/flydata/json/json_ext_spec.rb
 - spec/flydata/mysql/binlog_position_spec.rb
 - spec/flydata/mysql/table_ddl_spec.rb
 - spec/flydata/output/forwarder_spec.rb