fluent-plugin-bigquery 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 91a4bfe02680f2079998d36ab098c5498bdaea04
-  data.tar.gz: 234135f11479a6d364d1c5d0f58d90cfe7d72717
+  metadata.gz: 08a2d8a17b61a68a7b759d42d2c35c655660940e
+  data.tar.gz: 2e456086c9216056e9e4e7fe749521fa713ca2f5
 SHA512:
-  metadata.gz: a1e3df653dca491e163824f57e6d3d0932314ef59b2cc0cf36caa77f9007df624b183cd2b700caf5c02bd6a477322d136e44c14bed15f44c8c201eb9fb43838e
-  data.tar.gz: d20de3a18a7507fb5532480cd51704e23e79740d12a58bde482824095740681b10ede39a9e0611e878bba6e609078dc406e5e983b5ea5160e83b0632a4c83542
+  metadata.gz: fac063ee55019a58253ec7a4a0de9ae973c3e9d6814ee74455a010d0ed84382200aa0e9f1bfe9cbc45856c6756cd34ea85d85071156020acf27441a4ee61a2a4
+  data.tar.gz: d2a3365ca5515b02a35f3cfaae8f7a867b4e71f241441c08d4b71f6b0c41954943aa4161be0f67b2aa77ac41c9ab785f329d6eef1c37b6ddbc26a1e7352945d2

data/README.md

@@ -236,7 +236,7 @@ With this configuration, flushing will be done in 0.25 seconds after record inpu
 * support Load API
   * with automatically configured flush/buffer options
 * support optional data fields
-* support NULLABLE/REQUIRED/REPEATED field options
+* support NULLABLE/REQUIRED/REPEATED field options in field list style of configuration
 * OAuth installed application credentials support
 * Google API discovery expiration
 * Error classes

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -145,7 +145,7 @@ module Fluent
 
       @tablelist = @tables ? @tables.split(',') : [@table]
 
-      @fields = RecordSchema.new
+      @fields = RecordSchema.new('record')
       if @schema_path
         @fields.load_schema(JSON.parse(File.read(@schema_path)))
       end
@@ -317,7 +317,30 @@ module Fluent
     # end
 
     class FieldSchema
+      def initialize(name, mode = :nullable)
+        unless [:nullable, :required, :repeated].include?(mode)
+          raise ConfigError, "Unrecognized mode for #{name}: #{mode}"
+        end
+
+        @name = name
+        @mode = mode
+      end
+
+      attr_reader :name, :mode
+
       def format(value)
+        case @mode
+        when :nullable
+          format_one(value) unless value.nil?
+        when :required
+          raise "Required field #{name} cannot be null" if value.nil?
+          format_one(value)
+        when :repeated
+          value.nil? ? [] : value.map {|v| format_one(v) }
+        end
+      end
+
+      def format_one(value)
         raise NotImplementedError, "Must implement in a subclass" 
       end
     end
@@ -327,7 +350,7 @@ module Fluent
         :string
       end
 
-      def format(value)
+      def format_one(value)
         value.to_s
       end
     end
@@ -337,7 +360,7 @@ module Fluent
         :integer
       end
 
-      def format(value)
+      def format_one(value)
         value.to_i
       end
     end
@@ -347,7 +370,7 @@ module Fluent
         :float
       end
 
-      def format(value)
+      def format_one(value)
         value.to_f
       end
     end
@@ -357,7 +380,7 @@ module Fluent
         :boolean
       end
 
-      def format(value)
+      def format_one(value)
         !!value
       end
     end
@@ -367,7 +390,7 @@ module Fluent
         :timestamp
       end
 
-      def format(value)
+      def format_one(value)
         value
       end
     end
@@ -382,7 +405,8 @@ module Fluent
         :record => RecordSchema
       }.freeze
 
-      def initialize
+      def initialize(name, mode = :nullable)
+        super(name, mode)
         @fields = {}
       end
 
@@ -398,12 +422,15 @@ module Fluent
         schema.each do |field|
           raise ConfigError, 'field must have type' unless field.key?('type')
 
+          name = field['name']
+          mode = (field['mode'] || 'nullable').downcase.to_sym
+
           type = field['type'].downcase.to_sym
           field_schema_class = FIELD_TYPES[type]
           raise ConfigError, "Invalid field type: #{field['type']}" unless field_schema_class
 
-          field_schema = field_schema_class.new
-          @fields[field['name']] = field_schema
+          field_schema = field_schema_class.new(name, mode)
+          @fields[name] = field_schema
           if type == :record
             raise ConfigError, "record field must have fields" unless field.key?('fields')
             field_schema.load_schema(field['fields'])
@@ -423,16 +450,17 @@ module Fluent
         else
           schema = FIELD_TYPES[type]
           raise ConfigError, "[Bug] Invalid field type #{type}" unless schema
-          @fields[name] = schema.new
+          @fields[name] = schema.new(name)
         end
       end
 
-      def format(record)
+      def format_one(record)
         out = {}
         @fields.each do |key, schema|
           value = record[key]
-          next if value.nil? # field does not exists, or null value
-          out[key] = schema.format(value)
+          formatted = schema.format(value)
+          next if formatted.nil? # field does not exists, or null value
+          out[key] = formatted
         end
         out
       end
@@ -440,7 +468,7 @@ module Fluent
       private
       def register_record_field(name)
         if !@fields.key?(name)
-          @fields[name] = RecordSchema.new
+          @fields[name] = RecordSchema.new(name)
         else
           unless @fields[name].kind_of?(RecordSchema)
             raise ConfigError, "field #{name} is required to be a record but already registered as #{@field[name]}"

data/test/plugin/test_out_bigquery.rb

@@ -307,6 +307,84 @@ class BigQueryOutputTest < Test::Unit::TestCase
     assert_equal expected, MessagePack.unpack(buf)
   end
 
+  def test_format_repeated_field_with_schema
+    now = Time.now
+    input = [
+      now,
+      {
+        "tty" => nil,
+        "pwd" => "/home/yugui",
+        "user" => "fluentd",
+        "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
+      }
+    ]
+    expected = {
+      "json" => {
+        "time" => now.to_i,
+        "pwd" => "/home/yugui",
+        "user" => "fluentd",
+        "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
+      }
+    }
+
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      time_format %s
+      time_field  time
+
+      schema_path #{File.join(File.dirname(__FILE__), "testdata", "sudo.schema")}
+      field_integer time
+    CONFIG
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") { stub! }
+    end
+    driver.instance.start
+    buf = driver.instance.format_stream("my.tag", [input])
+    driver.instance.shutdown
+
+    assert_equal expected, MessagePack.unpack(buf)
+  end
+
+  def test_empty_value_in_required
+    now = Time.now
+    input = [
+      now,
+      {
+        "tty" => "pts/1",
+        "pwd" => "/home/yugui",
+        "user" => nil,
+        "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
+      }
+    ]
+
+    driver = create_driver(<<-CONFIG)
+      table foo
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      time_format %s
+      time_field  time
+
+      schema_path #{File.join(File.dirname(__FILE__), "testdata", "sudo.schema")}
+      field_integer time
+    CONFIG
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") { stub! }
+    end
+    driver.instance.start
+    assert_raises(RuntimeError, /cannot be null/) do
+      driver.instance.format_stream("my.tag", [input])
+    end
+    driver.instance.shutdown
+  end
+
   def test_write
     entry = {"json" => {"a" => "b"}}, {"json" => {"b" => "c"}}
     driver = create_driver(CONFIG)

data/test/plugin/testdata/sudo.schema

@@ -0,0 +1,27 @@
+[
+  {
+    "name": "time",
+    "type": "TIMESTAMP",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "tty",
+    "type": "STRING",
+    "mode": "NULLABLE"
+  },
+  {
+    "name": "pwd",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "user",
+    "type": "STRING",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "argv",
+    "type": "STRING",
+    "mode": "REPEATED"
+  }
+]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-12 00:00:00.000000000 Z
+date: 2014-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -143,6 +143,7 @@ files:
 - test/helper.rb
 - test/plugin/test_out_bigquery.rb
 - test/plugin/testdata/apache.schema
+- test/plugin/testdata/sudo.schema
 - test/test_load_request_body_wrapper.rb
 homepage: https://github.com/tagomoris/fluent-plugin-bigquery
 licenses:
@@ -172,5 +173,6 @@ test_files:
 - test/helper.rb
 - test/plugin/test_out_bigquery.rb
 - test/plugin/testdata/apache.schema
+- test/plugin/testdata/sudo.schema
 - test/test_load_request_body_wrapper.rb
 has_rdoc: