embulk-output-bigquery 0.6.3 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +3 -3
- data/embulk-output-bigquery.gemspec +1 -1
- data/lib/embulk/output/bigquery/value_converter_factory.rb +19 -0
- data/test/test_helper.rb +4 -1
- data/test/test_value_converter_factory.rb +45 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7ea0cf04e91a092e3d97bf3e46d1c181aab4943f
|
4
|
+
data.tar.gz: 60e970acbc16128189df8c274a832d23160e4c80
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 14fb288ad9781515a28cf72869ca7e76081202b8cf7e29d2448bb9ec37e6a8e8e73a046930f53c41785551b354a990ccc8ee2f9298b418de672c6dfaa2e6447b
|
7
|
+
data.tar.gz: 9f8f59c89cf7cc9974ab8a287ffbf522263ff5e81ff988de95f22c9086cf805d39d8f9c7c7d41310357464a29e5ec0404fe819b5ba41de36eb0d870c4ca31144
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -307,12 +307,12 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
|
|
307
307
|
|
308
308
|
- **column_options**: advanced: an array of options for columns
|
309
309
|
- **name**: column name
|
310
|
-
- **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, and `RECORD`. See belows for supported conversion type.
|
310
|
+
- **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, and `RECORD`. See belows for supported conversion type.
|
311
311
|
- boolean: `BOOLEAN`, `STRING` (default: `BOOLEAN`)
|
312
312
|
- long: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `INTEGER`)
|
313
313
|
- double: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `FLOAT`)
|
314
|
-
- string: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, `RECORD` (default: `STRING`)
|
315
|
-
- timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE` (default: `TIMESTAMP`)
|
314
|
+
- string: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, `RECORD` (default: `STRING`)
|
315
|
+
- timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE` (default: `TIMESTAMP`)
|
316
316
|
- json: `STRING`, `RECORD` (default: `STRING`)
|
317
317
|
- **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
|
318
318
|
- **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "embulk-output-bigquery"
|
3
|
-
spec.version = "0.6.
|
3
|
+
spec.version = "0.6.4"
|
4
4
|
spec.authors = ["Satoshi Akama", "Naotoshi Seo"]
|
5
5
|
spec.summary = "Google BigQuery output plugin for Embulk"
|
6
6
|
spec.description = "Embulk plugin that insert records to Google BigQuery."
|
@@ -210,6 +210,20 @@ module Embulk
|
|
210
210
|
TimeWithZone.set_zone_offset(Time.parse(val), zone_offset).strftime("%Y-%m-%d")
|
211
211
|
end
|
212
212
|
}
|
213
|
+
when 'DATETIME'
|
214
|
+
if @timestamp_format
|
215
|
+
Proc.new {|val|
|
216
|
+
next nil if val.nil?
|
217
|
+
with_typecast_error(val) do |val|
|
218
|
+
Time.strptime(val, @timestamp_format).strftime("%Y-%m-%d %H:%M:%S.%6N")
|
219
|
+
end
|
220
|
+
}
|
221
|
+
else
|
222
|
+
Proc.new {|val|
|
223
|
+
next nil if val.nil?
|
224
|
+
val # Users must care of BQ timestamp format
|
225
|
+
}
|
226
|
+
end
|
213
227
|
when 'RECORD'
|
214
228
|
Proc.new {|val|
|
215
229
|
next nil if val.nil?
|
@@ -252,6 +266,11 @@ module Embulk
|
|
252
266
|
next nil if val.nil?
|
253
267
|
val.localtime(zone_offset).strftime("%Y-%m-%d")
|
254
268
|
}
|
269
|
+
when 'DATETIME'
|
270
|
+
Proc.new {|val|
|
271
|
+
next nil if val.nil?
|
272
|
+
val.localtime(zone_offset).strftime("%Y-%m-%d %H:%M:%S.%6N")
|
273
|
+
}
|
255
274
|
else
|
256
275
|
raise NotSupportedType, "cannot take column type #{type} for timestamp column"
|
257
276
|
end
|
data/test/test_helper.rb
CHANGED
@@ -63,7 +63,8 @@ module Embulk
|
|
63
63
|
Column.new({index: 3, name: 'string', type: :string}),
|
64
64
|
Column.new({index: 4, name: 'timestamp', type: :timestamp}),
|
65
65
|
Column.new({index: 5, name: 'date', type: :timestamp}),
|
66
|
-
Column.new({index: 6, name: '
|
66
|
+
Column.new({index: 6, name: 'datetime', type: :timestamp}),
|
67
|
+
Column.new({index: 7, name: 'json', type: :json}),
|
67
68
|
])
|
68
69
|
task = {
|
69
70
|
'column_options' => [
|
@@ -73,6 +74,7 @@ module Embulk
|
|
73
74
|
{'name' => 'string', 'type' => 'INTEGER'},
|
74
75
|
{'name' => 'timestamp', 'type' => 'INTEGER'},
|
75
76
|
{'name' => 'date', 'type' => 'DATE'},
|
77
|
+
{'name' => 'datetime', 'type' => 'DATETIME'},
|
76
78
|
{'name' => 'json', 'type' => 'RECORD', 'fields' => [
|
77
79
|
{ 'name' => 'key1', 'type' => 'STRING' },
|
78
80
|
]},
|
@@ -85,6 +87,7 @@ module Embulk
|
|
85
87
|
{name: 'string', type: 'INTEGER'},
|
86
88
|
{name: 'timestamp', type: 'INTEGER'},
|
87
89
|
{name: 'date', type: 'DATE'},
|
90
|
+
{name: 'datetime', type: 'DATETIME'},
|
88
91
|
{name: 'json', type: 'RECORD', fields: [
|
89
92
|
{name: 'key1', type: 'STRING'},
|
90
93
|
]},
|
@@ -94,6 +94,10 @@ module Embulk
|
|
94
94
|
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
|
95
95
|
end
|
96
96
|
|
97
|
+
def test_datetime
|
98
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
|
99
|
+
end
|
100
|
+
|
97
101
|
def test_record
|
98
102
|
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
|
99
103
|
end
|
@@ -138,6 +142,10 @@ module Embulk
|
|
138
142
|
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
|
139
143
|
end
|
140
144
|
|
145
|
+
def test_datetime
|
146
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
|
147
|
+
end
|
148
|
+
|
141
149
|
def test_record
|
142
150
|
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
|
143
151
|
end
|
@@ -178,6 +186,10 @@ module Embulk
|
|
178
186
|
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
|
179
187
|
end
|
180
188
|
|
189
|
+
def test_datetime
|
190
|
+
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
|
191
|
+
end
|
192
|
+
|
181
193
|
def test_record
|
182
194
|
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
|
183
195
|
end
|
@@ -236,6 +248,20 @@ module Embulk
|
|
236
248
|
assert_raise { converter.call('foo') }
|
237
249
|
end
|
238
250
|
|
251
|
+
def test_datetime
|
252
|
+
converter = ValueConverterFactory.new(
|
253
|
+
SCHEMA_TYPE, 'DATETIME',
|
254
|
+
timestamp_format: '%Y/%m/%d'
|
255
|
+
).create_converter
|
256
|
+
assert_equal nil, converter.call(nil)
|
257
|
+
assert_equal "2016-02-26 00:00:00.000000", converter.call("2016/02/26")
|
258
|
+
|
259
|
+
# Users must care of BQ datetime format by themselves with no timestamp_format
|
260
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
|
261
|
+
assert_equal nil, converter.call(nil)
|
262
|
+
assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
|
263
|
+
end
|
264
|
+
|
239
265
|
def test_record
|
240
266
|
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
|
241
267
|
assert_equal({'foo'=>'foo'}, converter.call(%Q[{"foo":"foo"}]))
|
@@ -294,7 +320,7 @@ module Embulk
|
|
294
320
|
timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
|
295
321
|
expected = "2016-02-26"
|
296
322
|
assert_equal expected, converter.call(timestamp)
|
297
|
-
|
323
|
+
|
298
324
|
converter = ValueConverterFactory.new(
|
299
325
|
SCHEMA_TYPE, 'DATE', timezone: 'Asia/Tokyo'
|
300
326
|
).create_converter
|
@@ -306,6 +332,24 @@ module Embulk
|
|
306
332
|
assert_raise { converter.call('foo') }
|
307
333
|
end
|
308
334
|
|
335
|
+
def test_datetime
|
336
|
+
converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
|
337
|
+
assert_equal nil, converter.call(nil)
|
338
|
+
timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
|
339
|
+
expected = "2016-02-26 00:00:00.500000"
|
340
|
+
assert_equal expected, converter.call(timestamp)
|
341
|
+
|
342
|
+
converter = ValueConverterFactory.new(
|
343
|
+
SCHEMA_TYPE, 'DATETIME', timezone: 'Asia/Tokyo'
|
344
|
+
).create_converter
|
345
|
+
assert_equal nil, converter.call(nil)
|
346
|
+
timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00")
|
347
|
+
expected = "2016-02-26 00:00:00.500000"
|
348
|
+
assert_equal expected, converter.call(timestamp)
|
349
|
+
|
350
|
+
assert_raise { converter.call('foo') }
|
351
|
+
end
|
352
|
+
|
309
353
|
def test_record
|
310
354
|
assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
|
311
355
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Satoshi Akama
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-
|
12
|
+
date: 2019-11-06 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|