fluent-plugin-bigquery 3.0.0 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc6de961c8c42fddf3d9e297e93db560d16cfc098161232c90ee64f0a5679fee
-  data.tar.gz: 5ec1fee690f77d0fa25d8e427c6ad354cdfdbfafe30a4aee4fea9a5e73db5eb3
+  metadata.gz: 52e15b9cc1e5fba553895298e0e1a4510b2c3be0e333a8c9853ef8fb9a30e721
+  data.tar.gz: 9be1a5a48e75f63bd83c103111664690a7e8fa583dfda548a7da2dfd3437960f
 SHA512:
-  metadata.gz: 20fc96d420611a0d12f7cb34656ae87872f24131c70039383a8f8b7d51048a7d4f277a80675f2bee834113fd13d2a9780b772b517f2140481f7fb86ce63f24e3
-  data.tar.gz: cecc8f8682761ddfb22d942b69103823cc728923f6d7043d967254ed02c754db4e792132769f7f3aa91986aa27895ac83bf16358be21e03d3c94e77c43975231
+  metadata.gz: 8fd48a77fa9cf4b04706c4c3d041aa36ccc5011024fd6b37287c7ac661d0137458940e832410ae14a2385d77a0370908a22a6e856cbc9de4194da5a0866691ff
+  data.tar.gz: aff96e78358ced9a0a213739e8968bc4caa65afa1915ba4bc1a4660161978418ced12dbdec539ef960967f628c8893fd821db28ffe4e4401fe22010e200934ee

data/integration/fluent.conf

@@ -4,6 +4,14 @@
   bind 0.0.0.0
 </source>
 
+<source>
+  @type dummy
+  dummy {"json_field": {"foo": "val1", "bar": "val2", "hoge": 1}, "geography_field": {"type": "LineString", "coordinates": [[-118.4085, 33.9416], [-73.7781, 40.6413]]}, "timestamp_field": "2022-12-15T22:40:21+09:00", "date": "2022-12-15", "record_field": {"inner_field": "hoge", "inner_json": {"key1": "val1", "key2": "val2"}}, "repeated_string_field": ["a", "b", "c"]}
+  auto_increment_key id
+
+  tag insert_data
+</source>
+
 <match insert_data>
   @id bigquery-insert-integration
   @type bigquery_insert
@@ -21,7 +29,7 @@
     total_limit_size 1g
     path ./log/bigquery-insert-integration
 
-    flush_interval 30
+    flush_interval 15
     flush_thread_count 4
     flush_at_shutdown true
 
@@ -37,6 +45,7 @@
   dataset "#{ENV["DATASET_NAME"]}"
   table "#{ENV["TABLE_NAME"]}"
   auto_create_table false
+  # schema_path integration/schema.json
   fetch_schema true
   fetch_schema_table "#{ENV["TABLE_NAME"]}"
 
@@ -78,6 +87,7 @@
   dataset "#{ENV["DATASET_NAME"]}"
   table "#{ENV["TABLE_NAME"]}"
   auto_create_table false
+  # schema_path integration/schema.json
   fetch_schema true
   fetch_schema_table "#{ENV["TABLE_NAME"]}"
 

data/integration/schema.json

@@ -9,6 +9,16 @@
     "type": "STRING",
     "mode": "NULLABLE"
   },
+  {
+    "name": "json_field",
+    "type": "JSON",
+    "mode": "NULLABLE"
+  },
+  {
+    "name": "geography_field",
+    "type": "GEOGRAPHY",
+    "mode": "NULLABLE"
+  },
   {
     "name": "timestamp_field",
     "type": "TIMESTAMP",
@@ -18,5 +28,27 @@
     "name": "date",
     "type": "DATE",
     "mode": "REQUIRED"
+  },
+  {
+    "name": "record_field",
+    "type": "RECORD",
+    "mode": "NULLABLE",
+    "fields": [
+      {
+        "name": "inner_field",
+        "type": "STRING",
+        "mode": "REQUIRED"
+      },
+      {
+        "name": "inner_json",
+        "type": "JSON",
+        "mode": "REQUIRED"
+      }
+    ]
+  },
+  {
+    "name": "repeated_string_field",
+    "type": "STRING",
+    "mode": "REPEATED"
   }
 ]

data/lib/fluent/plugin/bigquery/schema.rb

@@ -23,23 +23,23 @@ module Fluent
 
       attr_reader :name, :mode
 
-      def format(value)
+      def format(value, is_load: false)
         case @mode
         when :nullable
-          format_one(value) unless value.nil?
+          format_one(value, is_load: is_load) unless value.nil?
         when :required
           if value.nil?
             log.warn "Required field #{name} cannot be null"
             nil
           else
-            format_one(value)
+            format_one(value, is_load: is_load)
           end
         when :repeated
-          value.nil? ? [] : value.each_with_object([]) { |v, arr| arr << format_one(v) if v }
+          value.nil? ? [] : value.each_with_object([]) { |v, arr| arr << format_one(v, is_load: true) if v }
         end
       end
 
-      def format_one(value)
+      def format_one(value, is_load: false)
         raise NotImplementedError, "Must implement in a subclass"
       end
 
@@ -57,7 +57,7 @@ module Fluent
         :string
       end
 
-      def format_one(value)
+      def format_one(value, is_load: false)
         if value.is_a?(Hash) || value.is_a?(Array)
           MultiJson.dump(value)
         else
@@ -66,12 +66,32 @@ module Fluent
       end
     end
 
+    class JsonFieldSchema < FieldSchema
+      def type
+        :json
+      end
+
+      def format_one(value, is_load: false)
+        if is_load
+          value
+        else
+          MultiJson.dump(value)
+        end
+      end
+    end
+
+    class GeographyFieldSchema < StringFieldSchema
+      def type
+        :geography
+      end
+    end
+
     class IntegerFieldSchema < FieldSchema
       def type
         :integer
       end
 
-      def format_one(value)
+      def format_one(value, is_load: false)
         value.to_i
       end
     end
@@ -81,7 +101,7 @@ module Fluent
         :float
       end
 
-      def format_one(value)
+      def format_one(value, is_load: false)
         value.to_f
       end
     end
@@ -91,7 +111,7 @@ module Fluent
         :numeric
       end
 
-      def format_one(value)
+      def format_one(value, is_load: false)
         value.to_s
       end
     end
@@ -101,7 +121,7 @@ module Fluent
         :boolean
       end
 
-      def format_one(value)
+      def format_one(value, is_load: false)
         !!value
       end
     end
@@ -114,7 +134,7 @@ module Fluent
         :timestamp
       end
 
-      def format_one(value)
+      def format_one(value, is_load: false)
         case value
         when Time
           value.strftime("%Y-%m-%d %H:%M:%S.%6L %:z")
@@ -137,7 +157,7 @@ module Fluent
         :date
       end
 
-      def format_one(value)
+      def format_one(value, is_load: false)
         if value.respond_to?(:strftime)
           value.strftime("%Y-%m-%d")
         else
@@ -151,7 +171,7 @@ module Fluent
         :datetime
       end
 
-      def format_one(value)
+      def format_one(value, is_load: false)
         if value.respond_to?(:strftime)
           value.strftime("%Y-%m-%dT%H:%M:%S.%6L")
         else
@@ -165,7 +185,7 @@ module Fluent
         :time
       end
 
-      def format_one(value)
+      def format_one(value, is_load: false)
         if value.respond_to?(:strftime)
           value.strftime("%H:%M:%S.%6L")
         else
@@ -185,6 +205,8 @@ module Fluent
         date: DateFieldSchema,
         datetime: DateTimeFieldSchema,
         time: TimeFieldSchema,
+        json: JsonFieldSchema,
+        geography: GeographyFieldSchema,
         record: RecordSchema
       }.freeze
 
@@ -256,12 +278,12 @@ module Fluent
         end
       end
 
-      def format_one(record)
+      def format_one(record, is_load: false)
         out = {}
         record.each do |key, value|
           next if value.nil?
           schema = @fields[key]
-          out[key] = schema ? schema.format(value) : value
+          out[key] = schema ? schema.format(value, is_load: is_load) : value
         end
         out
       end

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "3.0.0".freeze
+    VERSION = "3.1.0".freeze
   end
 end

data/lib/fluent/plugin/bigquery/writer.rb

@@ -347,7 +347,7 @@ module Fluent
           if try_count == 1
             # Table Not Found: Auto Create Table
             create_table(project, dataset, table_id, schema)
-          elsif try_count > 10
+          elsif try_count > 60 # timeout in about 300 seconds
             raise "A new table was created but it is not found."
           end
 

data/lib/fluent/plugin/out_bigquery_base.rb

@@ -172,7 +172,7 @@ module Fluent
           end
 
         begin
-          row = schema.format(record)
+          row = schema.format(record, is_load: !!@is_load)
           return if row.empty?
           @formatter.format(tag, time, row)
         rescue

data/lib/fluent/plugin/out_bigquery_insert.rb

@@ -46,6 +46,7 @@ module Fluent
 
       def configure(conf)
         super
+        @is_load = false
 
         if @insert_id_field
           if @insert_id_field !~ /^\$[\[\.]/ && @insert_id_field =~ /\./

data/lib/fluent/plugin/out_bigquery_load.rb

@@ -36,6 +36,7 @@ module Fluent
 
       def configure(conf)
         super
+        @is_load = true
 
         placeholder_params = "project=#{@project}/dataset=#{@dataset}/table=#{@tablelist.join(",")}/fetch_schema_table=#{@fetch_schema_table}"
         placeholder_validate!(:bigquery_load, placeholder_params)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.1.0
 platform: ruby
 authors:
 - Naoya Ito
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-05 00:00:00.000000000 Z
+date: 2022-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake