embulk-output-bigquery 0.6.3 → 0.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 8b3d7d7d675d8428946f81517d1002f667f4fafe
4
- data.tar.gz: 25940b93f70492675869d3c4dd50f83f8b7347cf
3
+ metadata.gz: 7ea0cf04e91a092e3d97bf3e46d1c181aab4943f
4
+ data.tar.gz: 60e970acbc16128189df8c274a832d23160e4c80
5
5
  SHA512:
6
- metadata.gz: 97a2aff66c765f24289717ac79e0a25a6bf31ee3ec5b84b64c96e8573382b31b0a27c30f06692a296b3bfedd70ea9f34f1a451cea7de27d3fa4c61a7502bab98
7
- data.tar.gz: b795d47af337e109dfafb9f41a0a720d0eb314c7ba7219193648505ec9dffa3874215b5d311256f625228a4f3e52b73153ee3d694a3d2f88d4c2fd0dd24960b1
6
+ metadata.gz: 14fb288ad9781515a28cf72869ca7e76081202b8cf7e29d2448bb9ec37e6a8e8e73a046930f53c41785551b354a990ccc8ee2f9298b418de672c6dfaa2e6447b
7
+ data.tar.gz: 9f8f59c89cf7cc9974ab8a287ffbf522263ff5e81ff988de95f22c9086cf805d39d8f9c7c7d41310357464a29e5ec0404fe819b5ba41de36eb0d870c4ca31144
@@ -1,3 +1,7 @@
1
+ ## 0.6.4 - 2019-11-06
2
+
3
+ * [enhancement] Add DATETIME type conveter (thanks to @kekekenta)
4
+
1
5
  ## 0.6.3 - 2019-10-28
2
6
 
3
7
  * [enhancement] Add DATE type conveter (thanks to @tksfjt1024)
data/README.md CHANGED
@@ -307,12 +307,12 @@ Column options are used to aid guessing BigQuery schema, or to define conversion
307
307
 
308
308
  - **column_options**: advanced: an array of options for columns
309
309
  - **name**: column name
310
- - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, and `RECORD`. See belows for supported conversion type.
310
+ - **type**: BigQuery type such as `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, and `RECORD`. See belows for supported conversion type.
311
311
  - boolean: `BOOLEAN`, `STRING` (default: `BOOLEAN`)
312
312
  - long: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `INTEGER`)
313
313
  - double: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP` (default: `FLOAT`)
314
- - string: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE`, `RECORD` (default: `STRING`)
315
- - timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATE` (default: `TIMESTAMP`)
314
+ - string: `BOOLEAN`, `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE`, `RECORD` (default: `STRING`)
315
+ - timestamp: `INTEGER`, `FLOAT`, `STRING`, `TIMESTAMP`, `DATETIME`, `DATE` (default: `TIMESTAMP`)
316
316
  - json: `STRING`, `RECORD` (default: `STRING`)
317
317
  - **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`)
318
318
  - **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column.
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = "embulk-output-bigquery"
3
- spec.version = "0.6.3"
3
+ spec.version = "0.6.4"
4
4
  spec.authors = ["Satoshi Akama", "Naotoshi Seo"]
5
5
  spec.summary = "Google BigQuery output plugin for Embulk"
6
6
  spec.description = "Embulk plugin that insert records to Google BigQuery."
@@ -210,6 +210,20 @@ module Embulk
210
210
  TimeWithZone.set_zone_offset(Time.parse(val), zone_offset).strftime("%Y-%m-%d")
211
211
  end
212
212
  }
213
+ when 'DATETIME'
214
+ if @timestamp_format
215
+ Proc.new {|val|
216
+ next nil if val.nil?
217
+ with_typecast_error(val) do |val|
218
+ Time.strptime(val, @timestamp_format).strftime("%Y-%m-%d %H:%M:%S.%6N")
219
+ end
220
+ }
221
+ else
222
+ Proc.new {|val|
223
+ next nil if val.nil?
224
+ val # Users must care of BQ timestamp format
225
+ }
226
+ end
213
227
  when 'RECORD'
214
228
  Proc.new {|val|
215
229
  next nil if val.nil?
@@ -252,6 +266,11 @@ module Embulk
252
266
  next nil if val.nil?
253
267
  val.localtime(zone_offset).strftime("%Y-%m-%d")
254
268
  }
269
+ when 'DATETIME'
270
+ Proc.new {|val|
271
+ next nil if val.nil?
272
+ val.localtime(zone_offset).strftime("%Y-%m-%d %H:%M:%S.%6N")
273
+ }
255
274
  else
256
275
  raise NotSupportedType, "cannot take column type #{type} for timestamp column"
257
276
  end
@@ -63,7 +63,8 @@ module Embulk
63
63
  Column.new({index: 3, name: 'string', type: :string}),
64
64
  Column.new({index: 4, name: 'timestamp', type: :timestamp}),
65
65
  Column.new({index: 5, name: 'date', type: :timestamp}),
66
- Column.new({index: 6, name: 'json', type: :json}),
66
+ Column.new({index: 6, name: 'datetime', type: :timestamp}),
67
+ Column.new({index: 7, name: 'json', type: :json}),
67
68
  ])
68
69
  task = {
69
70
  'column_options' => [
@@ -73,6 +74,7 @@ module Embulk
73
74
  {'name' => 'string', 'type' => 'INTEGER'},
74
75
  {'name' => 'timestamp', 'type' => 'INTEGER'},
75
76
  {'name' => 'date', 'type' => 'DATE'},
77
+ {'name' => 'datetime', 'type' => 'DATETIME'},
76
78
  {'name' => 'json', 'type' => 'RECORD', 'fields' => [
77
79
  { 'name' => 'key1', 'type' => 'STRING' },
78
80
  ]},
@@ -85,6 +87,7 @@ module Embulk
85
87
  {name: 'string', type: 'INTEGER'},
86
88
  {name: 'timestamp', type: 'INTEGER'},
87
89
  {name: 'date', type: 'DATE'},
90
+ {name: 'datetime', type: 'DATETIME'},
88
91
  {name: 'json', type: 'RECORD', fields: [
89
92
  {name: 'key1', type: 'STRING'},
90
93
  ]},
@@ -94,6 +94,10 @@ module Embulk
94
94
  assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
95
95
  end
96
96
 
97
+ def test_datetime
98
+ assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
99
+ end
100
+
97
101
  def test_record
98
102
  assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
99
103
  end
@@ -138,6 +142,10 @@ module Embulk
138
142
  assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
139
143
  end
140
144
 
145
+ def test_datetime
146
+ assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
147
+ end
148
+
141
149
  def test_record
142
150
  assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
143
151
  end
@@ -178,6 +186,10 @@ module Embulk
178
186
  assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
179
187
  end
180
188
 
189
+ def test_datetime
190
+ assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter }
191
+ end
192
+
181
193
  def test_record
182
194
  assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
183
195
  end
@@ -236,6 +248,20 @@ module Embulk
236
248
  assert_raise { converter.call('foo') }
237
249
  end
238
250
 
251
+ def test_datetime
252
+ converter = ValueConverterFactory.new(
253
+ SCHEMA_TYPE, 'DATETIME',
254
+ timestamp_format: '%Y/%m/%d'
255
+ ).create_converter
256
+ assert_equal nil, converter.call(nil)
257
+ assert_equal "2016-02-26 00:00:00.000000", converter.call("2016/02/26")
258
+
259
+ # Users must care of BQ datetime format by themselves with no timestamp_format
260
+ converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
261
+ assert_equal nil, converter.call(nil)
262
+ assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
263
+ end
264
+
239
265
  def test_record
240
266
  converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
241
267
  assert_equal({'foo'=>'foo'}, converter.call(%Q[{"foo":"foo"}]))
@@ -294,7 +320,7 @@ module Embulk
294
320
  timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
295
321
  expected = "2016-02-26"
296
322
  assert_equal expected, converter.call(timestamp)
297
-
323
+
298
324
  converter = ValueConverterFactory.new(
299
325
  SCHEMA_TYPE, 'DATE', timezone: 'Asia/Tokyo'
300
326
  ).create_converter
@@ -306,6 +332,24 @@ module Embulk
306
332
  assert_raise { converter.call('foo') }
307
333
  end
308
334
 
335
+ def test_datetime
336
+ converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATETIME').create_converter
337
+ assert_equal nil, converter.call(nil)
338
+ timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
339
+ expected = "2016-02-26 00:00:00.500000"
340
+ assert_equal expected, converter.call(timestamp)
341
+
342
+ converter = ValueConverterFactory.new(
343
+ SCHEMA_TYPE, 'DATETIME', timezone: 'Asia/Tokyo'
344
+ ).create_converter
345
+ assert_equal nil, converter.call(nil)
346
+ timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00")
347
+ expected = "2016-02-26 00:00:00.500000"
348
+ assert_equal expected, converter.call(timestamp)
349
+
350
+ assert_raise { converter.call('foo') }
351
+ end
352
+
309
353
  def test_record
310
354
  assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
311
355
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-bigquery
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.3
4
+ version: 0.6.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Satoshi Akama
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2019-10-28 00:00:00.000000000 Z
12
+ date: 2019-11-06 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement