embulk-output-bigquery 0.4.14 → 0.6.3

data/test/test_value_converter_factory.rb

@@ -90,6 +90,10 @@ module Embulk
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -130,6 +134,10 @@ module Embulk
           assert_equal 1408452095, converter.call(1408452095)
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -166,6 +174,10 @@ module Embulk
           assert_equal 1408452095.188766, converter.call(1408452095.188766)
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -216,6 +228,14 @@ module Embulk
           assert_equal "2016-02-26 00:00:00", converter.call("2016-02-26 00:00:00")
         end
 
+        def test_date
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter
+          assert_equal nil, converter.call(nil)
+          assert_equal "2016-02-26", converter.call("2016-02-26")
+          assert_equal "2016-02-26", converter.call("2016-02-26 00:00:00")
+          assert_raise { converter.call('foo') }
+        end
+
         def test_record
           converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
           assert_equal({'foo'=>'foo'}, converter.call(%Q[{"foo":"foo"}]))
@@ -268,6 +288,24 @@ module Embulk
           assert_equal expected, converter.call(Time.at(subject).utc)
         end
 
+        def test_date
+          converter = ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-26 00:00:00.500000 +00:00")
+          expected = "2016-02-26"
+          assert_equal expected, converter.call(timestamp)
+          
+          converter = ValueConverterFactory.new(
+            SCHEMA_TYPE, 'DATE', timezone: 'Asia/Tokyo'
+          ).create_converter
+          assert_equal nil, converter.call(nil)
+          timestamp = Time.parse("2016-02-25 15:00:00.500000 +00:00")
+          expected = "2016-02-26"
+          assert_equal expected, converter.call(timestamp)
+
+          assert_raise { converter.call('foo') }
+        end
+
         def test_record
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter }
         end
@@ -298,6 +336,10 @@ module Embulk
           assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'TIMESTAMP').create_converter }
         end
 
+        def test_date
+          assert_raise { ValueConverterFactory.new(SCHEMA_TYPE, 'DATE').create_converter }
+        end
+
         def test_record
           converter = ValueConverterFactory.new(SCHEMA_TYPE, 'RECORD').create_converter
           assert_equal nil, converter.call(nil)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.4.14
+  version: 0.6.3
 platform: ruby
 authors:
 - Satoshi Akama
@@ -9,22 +9,42 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-08-10 00:00:00.000000000 Z
+date: 2019-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+  name: signet
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.33.0
   name: google-api-client
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.33.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -83,52 +103,8 @@ files:
 - README.md
 - Rakefile
 - embulk-output-bigquery.gemspec
-- example/config_append_direct_schema_update_options.yml
-- example/config_client_options.yml
-- example/config_csv.yml
-- example/config_delete_in_advance.yml
-- example/config_delete_in_advance_field_partitioned_table.yml
-- example/config_delete_in_advance_partitioned_table.yml
-- example/config_expose_errors.yml
-- example/config_gcs.yml
-- example/config_guess_from_embulk_schema.yml
-- example/config_guess_with_column_options.yml
-- example/config_gzip.yml
-- example/config_jsonl.yml
-- example/config_max_threads.yml
-- example/config_min_ouput_tasks.yml
-- example/config_mode_append.yml
-- example/config_mode_append_direct.yml
-- example/config_nested_record.yml
-- example/config_payload_column.yml
-- example/config_payload_column_index.yml
-- example/config_prevent_duplicate_insert.yml
-- example/config_progress_log_interval.yml
-- example/config_replace.yml
-- example/config_replace_backup.yml
-- example/config_replace_backup_field_partitioned_table.yml
-- example/config_replace_backup_partitioned_table.yml
-- example/config_replace_field_partitioned_table.yml
-- example/config_replace_partitioned_table.yml
-- example/config_replace_schema_update_options.yml
-- example/config_skip_file_generation.yml
-- example/config_table_strftime.yml
-- example/config_template_table.yml
-- example/config_uncompressed.yml
-- example/config_with_rehearsal.yml
-- example/example.csv
-- example/example.yml
-- example/example2_1.csv
-- example/example2_2.csv
-- example/example4_1.csv
-- example/example4_2.csv
-- example/example4_3.csv
-- example/example4_4.csv
-- example/json_key.json
-- example/nested_example.jsonl
-- example/schema.json
-- example/schema_expose_errors.json
 - lib/embulk/output/bigquery.rb
+- lib/embulk/output/bigquery/auth.rb
 - lib/embulk/output/bigquery/bigquery_client.rb
 - lib/embulk/output/bigquery/file_writer.rb
 - lib/embulk/output/bigquery/gcs_client.rb
@@ -162,7 +138,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project:
+rubygems_version: 2.6.14.1
 signing_key:
 specification_version: 4
 summary: Google BigQuery output plugin for Embulk

data/example/config_append_direct_schema_update_options.yml

@@ -1,31 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: date,        type: string}
-      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
-      - {name: "null",      type: string}
-      - {name: long,        type: long}
-      - {name: string,      type: string}
-      - {name: double,     type: double}
-      - {name: boolean,     type: boolean}
-out:
-  type: bigquery
-  mode: append_direct
-  auth_method: json_key
-  json_keyfile: example/your-project-000.json
-  dataset: your_dataset_name
-  table: your_table_name
-  source_format: NEWLINE_DELIMITED_JSON
-  compression: NONE
-  auto_create_dataset: true
-  auto_create_table: true
-  schema_file: example/schema.json
-  schema_update_options: [ALLOW_FIELD_ADDITION, ALLOW_FIELD_RELAXATION]

data/example/config_client_options.yml

@@ -1,33 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: date,        type: string}
-      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
-      - {name: "null",      type: string}
-      - {name: long,        type: long}
-      - {name: string,      type: string}
-      - {name: double,      type: double}
-      - {name: boolean,     type: boolean}
-out:
-  type: bigquery
-  mode: replace
-  auth_method: json_key
-  json_keyfile: example/your-project-000.json
-  dataset: your_dataset_name
-  table: your_table_name
-  source_format: NEWLINE_DELIMITED_JSON
-  auto_create_dataset: true
-  auto_create_table: true
-  schema_file: example/schema.json
-  timeout_sec: 400
-  open_timeout_sec: 400
-  retries: 2
-  application_name: "Embulk BigQuery plugin test"

data/example/config_csv.yml

@@ -1,30 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: date,        type: string}
-      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
-      - {name: "null",      type: string}
-      - {name: long,        type: long}
-      - {name: string,      type: string}
-      - {name: double,      type: double}
-      - {name: boolean,     type: boolean}
-out:
-  type: bigquery
-  mode: replace
-  auth_method: json_key
-  json_keyfile: example/your-project-000.json
-  dataset: your_dataset_name
-  table: your_table_name
-  source_format: CSV
-  compression: GZIP
-  auto_create_dataset: true
-  auto_create_table: true
-  schema_file: example/schema.json

data/example/config_delete_in_advance.yml

@@ -1,29 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: date,        type: string}
-      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
-      - {name: "null",      type: string}
-      - {name: long,        type: long}
-      - {name: string,      type: string}
-      - {name: double,      type: double}
-      - {name: boolean,     type: boolean}
-out:
-  type: bigquery
-  mode: delete_in_advance
-  auth_method: json_key
-  json_keyfile: example/your-project-000.json
-  dataset: your_dataset_name
-  table: your_table_name
-  source_format: NEWLINE_DELIMITED_JSON
-  auto_create_dataset: true
-  auto_create_table: true
-  schema_file: example/schema.json

data/example/config_delete_in_advance_field_partitioned_table.yml

@@ -1,33 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: date,        type: string}
-      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
-      - {name: "null",      type: string}
-      - {name: long,        type: long}
-      - {name: string,      type: string}
-      - {name: double,      type: double}
-      - {name: boolean,     type: boolean}
-out:
-  type: bigquery
-  mode: delete_in_advance
-  auth_method: json_key
-  json_keyfile: example/your-project-000.json
-  dataset: your_dataset_name
-  table: your_field_partitioned_table_name
-  source_format: NEWLINE_DELIMITED_JSON
-  compression: NONE
-  auto_create_dataset: true
-  auto_create_table: true
-  schema_file: example/schema.json
-  time_partitioning:
-    type: 'DAY'
-    field: timestamp

data/example/config_delete_in_advance_partitioned_table.yml

@@ -1,33 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: date,        type: string}
-      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
-      - {name: "null",      type: string}
-      - {name: long,        type: long}
-      - {name: string,      type: string}
-      - {name: double,      type: double}
-      - {name: boolean,     type: boolean}
-out:
-  type: bigquery
-  mode: delete_in_advance
-  auth_method: json_key
-  json_keyfile: example/your-project-000.json
-  dataset: your_dataset_name
-  table: your_partitioned_table_name$20160929
-  source_format: NEWLINE_DELIMITED_JSON
-  compression: NONE
-  auto_create_dataset: true
-  auto_create_table: true
-  schema_file: example/schema.json
-  time_partitioning:
-    type: 'DAY'
-    expiration_ms: 100

data/example/config_expose_errors.yml

@@ -1,30 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: date,        type: string}
-      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
-      - {name: "null",      type: string}
-      - {name: long,        type: long}
-      - {name: string,      type: string}
-      - {name: double,      type: double}
-      - {name: boolean,     type: boolean}
-out:
-  type: bigquery
-  mode: replace
-  auth_method: json_key
-  json_keyfile: example/your-project-000.json
-  dataset: your_dataset_name
-  table: your_table_name
-  source_format: NEWLINE_DELIMITED_JSON
-  compression: NONE
-  auto_create_dataset: true
-  auto_create_table: true
-  schema_file: example/schema_expose_errors.json

data/example/config_gcs.yml

@@ -1,32 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: date,        type: string}
-      - {name: timestamp,   type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
-      - {name: "null",      type: string}
-      - {name: long,        type: long}
-      - {name: string,      type: string}
-      - {name: double,      type: double}
-      - {name: boolean,     type: boolean}
-out:
-  type: bigquery
-  mode: replace
-  auth_method: json_key
-  json_keyfile: example/your-project-000.json
-  dataset: your_dataset_name
-  table: your_table_name
-  source_format: NEWLINE_DELIMITED_JSON
-  compression: GZIP
-  auto_create_dataset: true
-  auto_create_table: true
-  schema_file: example/schema.json
-  gcs_bucket: your_bucket_name
-  auto_create_gcs_bucket: true