flydata 0.5.15 → 0.5.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/ext/flydata/json/json_ext.cpp +152 -11
- data/flydata.gemspec +0 -0
- data/spec/flydata/json/json_ext_spec.rb +69 -0
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +12 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 145b738aa44e8337c13b426d551c12d62d3f688b
|
4
|
+
data.tar.gz: 9597729174fa3a8d1e55eed087eb77ebd3e39cfd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 62a90057efd34adb6be5187d932d53d961f80f4e3f6138baa8dba840aaa0e91e22be96ba58fb5ce117e89d74048a4eb234244e59dde2fbe1ff4d86be3f246ffa
|
7
|
+
data.tar.gz: b0eb15b12279ea4865ef9bd93835bbddbb25a4b7d6b04d1711d14a470e6265224e7ded1c733866c9ea93fc1846e6327d5b116181611170ed54da151685eb59bf
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.16
|
@@ -10,6 +10,16 @@ _declspec(dllexport)
|
|
10
10
|
void Init_json_ext();
|
11
11
|
}
|
12
12
|
|
13
|
+
inline void write_escape_str(const char*& buf, long& buf_len, const char* escape_str, long escape_str_len, std::stringstream& ss)
|
14
|
+
{
|
15
|
+
if (buf_len > 0)
|
16
|
+
{
|
17
|
+
ss.write(buf, buf_len);
|
18
|
+
}
|
19
|
+
ss.write(escape_str, escape_str_len);
|
20
|
+
buf = 0; buf_len = 0;
|
21
|
+
}
|
22
|
+
|
13
23
|
inline void add_json_value(const char* buf, long buf_len, bool quote, std::stringstream& ss)
|
14
24
|
{
|
15
25
|
if (quote)
|
@@ -21,31 +31,162 @@ inline void add_json_value(const char* buf, long buf_len, bool quote, std::strin
|
|
21
31
|
const char* esc_str = 0;
|
22
32
|
for (long i = 0; i < buf_len; i++) {
|
23
33
|
if (!ptr) { ptr = buf + i; len = 0; }
|
34
|
+
|
35
|
+
if ((buf[i] >= 0x5d && buf[i] < 0x7f) || // small letter alphabet
|
36
|
+
(buf[i] >= 0x30 && buf[i] < 0x5c) || // digit and capital letter alphabet
|
37
|
+
(buf[i] >= 0x80) || // multi-byte utf-8
|
38
|
+
(buf[i] >= 0x23 && buf[i] < 0x2f) || // signs
|
39
|
+
(buf[i] == 0x21) // exclamation mark
|
40
|
+
) {
|
41
|
+
len++;
|
42
|
+
continue;
|
43
|
+
}
|
44
|
+
// Handle JSON special characters. See http://www.json.org/
|
24
45
|
switch (buf[i])
|
25
46
|
{
|
26
47
|
case '\"':
|
27
48
|
esc_str = "\\\"";
|
49
|
+
write_escape_str(ptr, len, esc_str, 2, ss);
|
50
|
+
break;
|
28
51
|
case '\\':
|
29
52
|
esc_str = "\\\\";
|
53
|
+
write_escape_str(ptr, len, esc_str, 2, ss);
|
54
|
+
break;
|
30
55
|
case '/':
|
31
56
|
esc_str = "\\/";
|
57
|
+
write_escape_str(ptr, len, esc_str, 2, ss);
|
58
|
+
break;
|
59
|
+
case '\t':
|
60
|
+
esc_str = "\\t";
|
61
|
+
write_escape_str(ptr, len, esc_str, 2, ss);
|
62
|
+
break;
|
63
|
+
case '\n':
|
64
|
+
esc_str = "\\n";
|
65
|
+
write_escape_str(ptr, len, esc_str, 2, ss);
|
66
|
+
break;
|
67
|
+
case 0x00:
|
68
|
+
esc_str = "\\u0000";
|
69
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
70
|
+
break;
|
71
|
+
case 0x01:
|
72
|
+
esc_str = "\\u0001";
|
73
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
74
|
+
break;
|
75
|
+
case 0x02:
|
76
|
+
esc_str = "\\u0002";
|
77
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
78
|
+
break;
|
79
|
+
case 0x03:
|
80
|
+
esc_str = "\\u0003";
|
81
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
82
|
+
break;
|
83
|
+
case 0x04:
|
84
|
+
esc_str = "\\u0004";
|
85
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
86
|
+
break;
|
87
|
+
case 0x05:
|
88
|
+
esc_str = "\\u0005";
|
89
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
90
|
+
break;
|
91
|
+
case 0x06:
|
92
|
+
esc_str = "\\u0006";
|
93
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
94
|
+
break;
|
95
|
+
case '\a':
|
96
|
+
esc_str = "\\u0007";
|
97
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
98
|
+
break;
|
32
99
|
case '\b':
|
33
100
|
esc_str = "\\b";
|
101
|
+
write_escape_str(ptr, len, esc_str, 2, ss);
|
102
|
+
break;
|
103
|
+
case 0x0b:
|
104
|
+
esc_str = "\\u000b";
|
105
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
106
|
+
break;
|
34
107
|
case '\f':
|
35
108
|
esc_str = "\\f";
|
36
|
-
|
37
|
-
|
109
|
+
write_escape_str(ptr, len, esc_str, 2, ss);
|
110
|
+
break;
|
38
111
|
case '\r':
|
39
112
|
esc_str = "\\r";
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
ptr
|
113
|
+
write_escape_str(ptr, len, esc_str, 2, ss);
|
114
|
+
break;
|
115
|
+
case 0x0e:
|
116
|
+
esc_str = "\\u000e";
|
117
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
118
|
+
break;
|
119
|
+
case 0x0f:
|
120
|
+
esc_str = "\\u000f";
|
121
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
122
|
+
break;
|
123
|
+
case 0x10:
|
124
|
+
esc_str = "\\u0010";
|
125
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
126
|
+
break;
|
127
|
+
case 0x11:
|
128
|
+
esc_str = "\\u0011";
|
129
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
130
|
+
break;
|
131
|
+
case 0x12:
|
132
|
+
esc_str = "\\u0012";
|
133
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
134
|
+
break;
|
135
|
+
case 0x13:
|
136
|
+
esc_str = "\\u0013";
|
137
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
138
|
+
break;
|
139
|
+
case 0x14:
|
140
|
+
esc_str = "\\u0014";
|
141
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
142
|
+
break;
|
143
|
+
case 0x15:
|
144
|
+
esc_str = "\\u0015";
|
145
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
146
|
+
break;
|
147
|
+
case 0x16:
|
148
|
+
esc_str = "\\u0016";
|
149
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
150
|
+
break;
|
151
|
+
case 0x17:
|
152
|
+
esc_str = "\\u0017";
|
153
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
154
|
+
break;
|
155
|
+
case 0x18:
|
156
|
+
esc_str = "\\u0018";
|
157
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
158
|
+
break;
|
159
|
+
case 0x19:
|
160
|
+
esc_str = "\\u0019";
|
161
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
162
|
+
break;
|
163
|
+
case 0x1a:
|
164
|
+
esc_str = "\\u001a";
|
165
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
166
|
+
break;
|
167
|
+
case 0x1b:
|
168
|
+
esc_str = "\\u001b";
|
169
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
170
|
+
break;
|
171
|
+
case 0x1c:
|
172
|
+
esc_str = "\\u001c";
|
173
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
174
|
+
break;
|
175
|
+
case 0x1d:
|
176
|
+
esc_str = "\\u001d";
|
177
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
178
|
+
break;
|
179
|
+
case 0x1e:
|
180
|
+
esc_str = "\\u001e";
|
181
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
182
|
+
break;
|
183
|
+
case 0x1f:
|
184
|
+
esc_str = "\\u001f";
|
185
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
186
|
+
break;
|
187
|
+
case 0x7f:
|
188
|
+
esc_str = "\\u007f";
|
189
|
+
write_escape_str(ptr, len, esc_str, 6, ss);
|
49
190
|
break;
|
50
191
|
default:
|
51
192
|
len++;
|
data/flydata.gemspec
CHANGED
Binary file
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata/json/json_ext'
|
3
|
+
|
4
|
+
describe JSON do
|
5
|
+
describe '.generate_kv_pairs' do
|
6
|
+
subject { described_class.generate_kv_pairs(keys, values) }
|
7
|
+
|
8
|
+
context 'testing values' do
|
9
|
+
let(:keys) { ['test'] }
|
10
|
+
let(:values) { [ value ] }
|
11
|
+
let(:expected_result) { %Q|{"test":"#{expected_value}"}| }
|
12
|
+
shared_examples 'producing the expected result' do
|
13
|
+
it do
|
14
|
+
is_expected.to eq expected_result
|
15
|
+
end
|
16
|
+
end
|
17
|
+
context 'regular string' do
|
18
|
+
let(:value) { "a regular text" }
|
19
|
+
let(:expected_value) { value }
|
20
|
+
it_behaves_like "producing the expected result"
|
21
|
+
end
|
22
|
+
context 'regular UTF-8 2 byte chars' do
|
23
|
+
let(:value) { "¢Àø" }
|
24
|
+
let(:expected_value) { value }
|
25
|
+
it_behaves_like "producing the expected result"
|
26
|
+
end
|
27
|
+
context 'regular UTF-8 3 byte chars' do
|
28
|
+
let(:value) { "三国一" }
|
29
|
+
let(:expected_value) { value }
|
30
|
+
it_behaves_like "producing the expected result"
|
31
|
+
end
|
32
|
+
context '/' do
|
33
|
+
let(:value) { "/" }
|
34
|
+
let(:expected_value) { "\\/" }
|
35
|
+
it_behaves_like "producing the expected result"
|
36
|
+
end
|
37
|
+
context '\\' do
|
38
|
+
let(:value) { "\\" }
|
39
|
+
let(:expected_value) { "\\\\" }
|
40
|
+
it_behaves_like "producing the expected result"
|
41
|
+
end
|
42
|
+
context '"' do
|
43
|
+
let(:value) { "\"" }
|
44
|
+
let(:expected_value) { "\\\"" }
|
45
|
+
it_behaves_like "producing the expected result"
|
46
|
+
end
|
47
|
+
context '\'' do
|
48
|
+
let(:value) { "'" }
|
49
|
+
let(:expected_value) { "'" }
|
50
|
+
it_behaves_like "producing the expected result"
|
51
|
+
end
|
52
|
+
context '\b' do
|
53
|
+
let(:value) { "\b" }
|
54
|
+
let(:expected_value) { "\\b" }
|
55
|
+
it_behaves_like "producing the expected result"
|
56
|
+
end
|
57
|
+
context '\x02' do
|
58
|
+
let(:value) { "\x02" }
|
59
|
+
let(:expected_value) { "\\u0002" }
|
60
|
+
it_behaves_like "producing the expected result"
|
61
|
+
end
|
62
|
+
context '^\\' do
|
63
|
+
let(:value) { "\x1c" }
|
64
|
+
let(:expected_value) { "\\u001c" }
|
65
|
+
it_behaves_like "producing the expected result"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -1068,6 +1068,18 @@ EOS
|
|
1068
1068
|
end
|
1069
1069
|
end
|
1070
1070
|
|
1071
|
+
context 'when ^B is given' do
|
1072
|
+
let(:target_line) {"INSERT INTO `test_table` VALUES ('testtest');"}
|
1073
|
+
it 'should not remove the leading zeros' do
|
1074
|
+
expect(subject).to eq([["test\x02test"]])
|
1075
|
+
end
|
1076
|
+
end
|
1077
|
+
context 'when ^\\ is given' do
|
1078
|
+
let(:target_line) {"INSERT INTO `test_table` VALUES ('testtest');"}
|
1079
|
+
it 'should not remove the leading zeros' do
|
1080
|
+
expect(subject).to eq([["test\x1ctest"]])
|
1081
|
+
end
|
1082
|
+
end
|
1071
1083
|
end
|
1072
1084
|
end
|
1073
1085
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.16
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2015-
|
15
|
+
date: 2015-11-02 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -696,6 +696,7 @@ files:
|
|
696
696
|
- spec/flydata/helper/scheduler_spec.rb
|
697
697
|
- spec/flydata/helper/worker_spec.rb
|
698
698
|
- spec/flydata/heroku_spec.rb
|
699
|
+
- spec/flydata/json/json_ext_spec.rb
|
699
700
|
- spec/flydata/mysql/binlog_position_spec.rb
|
700
701
|
- spec/flydata/mysql/table_ddl_spec.rb
|
701
702
|
- spec/flydata/output/forwarder_spec.rb
|