fluent-plugin-bigquery 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 793b3fbd0189044538497bcef9dc244adf987d86
-  data.tar.gz: 115ff35e20bf3e3fe58e54978f7e659678e14a17
+  metadata.gz: caee66954ca996b43797e050825d5115dcd132f8
+  data.tar.gz: 040afb42fa0fe48f61a90d11e9ea5684d170974c
 SHA512:
-  metadata.gz: d230732372df108fbcdf59aec5485f2837028531fb1cb9edf0582dfc31654a747bb35494e9de646067e295960709e778d9e69f41f80f1a8810e122b952c971e4
-  data.tar.gz: fc7dbd59a34a44f9f8f2aef3829fc352636c58a5fad3d8ddbf661368c912d8a2ba5d6f2080c66fc85c32f615916c3ad980f3e4e3a9b6e967bc8bfc1fc5712e2c
+  metadata.gz: 15aed88d76491c467ff6e865381d1d78884d632f998c71569e53faadb1acf18d572e4459b55d4ff5a3e57121365cf1ec90c4c292726e492bcfdac6a52ab7f815
+  data.tar.gz: a5ed1a27a117a6f54a2931741c2ffdf37b856e54074af94e2740e70ed361fa21f4790a20e926d675019d1280574497303825562be0b75097200b5e58dd49646c

data/README.md

@@ -1,4 +1,4 @@
-# fluent-plugin-webhdfs
+# fluent-plugin-bigquery
 
 Fluentd output plugin to load/insert data into Google BigQuery.
 
@@ -131,8 +131,7 @@ With this configuration, flushing will be done in 0.25 seconds after record inpu
 
 * support Load API
   * with automatically configured flush/buffer options
-* support RECORD field
-  * and support optional data fields
+* support optional data fields
 * support NULLABLE/REQUIRED/REPEATED field options
 * OAuth installed application credentials support
 * Google API discovery expiration

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -126,25 +126,25 @@ module Fluent
 
       @tablelist = @tables ? @tables.split(',') : [@table]
 
-      @fields = {}
+      @fields = RecordSchema.new
       if @field_string
         @field_string.split(',').each do |fieldname|
-          @fields[fieldname] = :string
+          @fields.register_field fieldname, :string
         end
       end
       if @field_integer
         @field_integer.split(',').each do |fieldname|
-          @fields[fieldname] = :integer
+          @fields.register_field fieldname, :integer
         end
       end
       if @field_float
         @field_float.split(',').each do |fieldname|
-          @fields[fieldname] = :float
+          @fields.register_field fieldname, :float
         end
       end
       if @field_boolean
         @field_boolean.split(',').each do |fieldname|
-          @fields[fieldname] = :boolean
+          @fields.register_field fieldname, :boolean
         end
       end
 
@@ -204,7 +204,7 @@ module Fluent
           "rows" => rows
         }
       )
-      if res.status != 200
+      unless res.success?
         # api_error? -> client cache clear
         @cached_client = nil
 
@@ -227,32 +227,14 @@ module Fluent
       raise NotImplementedError # TODO
     end
 
-    def format_record(record)
-      out = {}
-      @fields.each do |key, type|
-        value = record[key]
-        next if value.nil? # field does not exists, or null value
-        out[key] = case type
-                   when :string  then record[key].to_s
-                   when :integer then record[key].to_i
-                   when :float   then record[key].to_f
-                   when :boolean then !!record[key]
-                   # when :record
-                   else
-                     raise "BUG: unknown field type #{type}"
-                   end
-      end
-      out
-    end
-
     def format_stream(tag, es)
       super
       buf = ''
       es.each do |time, record|
         row = if @time_field
-                format_record(record.merge({@time_field => @timef.format(time)}))
+                @fields.format(record.merge({@time_field => @timef.format(time)}))
               else
-                format_record(record)
+                @fields.format(record)
               end
         buf << {"json" => row}.to_msgpack unless row.empty?
       end
@@ -292,5 +274,107 @@ module Fluent
     #   client.authorization = flow.authorize # browser authentication !
     #   client
     # end
+
+    class FieldSchema
+      def format(value)
+        raise NotImplementedError, "Must implement in a subclass" 
+      end
+    end
+
+    class StringFieldSchema < FieldSchema
+      def type
+        :string
+      end
+
+      def format(value)
+        value.to_s
+      end
+    end
+
+    class IntegerFieldSchema < FieldSchema
+      def type
+        :integer
+      end
+
+      def format(value)
+        value.to_i
+      end
+    end
+
+    class FloatFieldSchema < FieldSchema
+      def type
+        :float
+      end
+
+      def format(value)
+        value.to_f
+      end
+    end
+
+    class BooleanFieldSchema < FieldSchema
+      def type
+        :boolean
+      end
+
+      def format(value)
+        !!value
+      end
+    end
+
+    class RecordSchema < FieldSchema
+      FIELD_TYPES = {
+        :string => StringFieldSchema,
+        :integer => IntegerFieldSchema,
+        :float => FloatFieldSchema,
+        :boolean => BooleanFieldSchema
+      }.freeze
+
+      def initialize
+        @fields = {}
+      end
+
+      def type
+        :record
+      end
+
+      def [](name)
+        @fields[name]
+      end
+
+      def register_field(name, type)
+        raise ConfigError, "field #{name} is registered twice" if @fields.key?(name)
+        if name[/\./]
+          recordname = $`
+          fieldname = $'
+          register_record_field(recordname)
+          @fields[recordname].register_field(fieldname, type)
+        else
+          schema = FIELD_TYPES[type]
+          raise ConfigError, "[Bug] Invalid field type #{type}" unless schema
+          @fields[name] = schema.new
+        end
+      end
+
+      def format(record)
+        out = {}
+        @fields.each do |key, schema|
+          value = record[key]
+          next if value.nil? # field does not exists, or null value
+          out[key] = schema.format(value)
+        end
+        out
+      end
+
+      private
+      def register_record_field(name)
+        if !@fields.key?(name)
+          @fields[name] = RecordSchema.new
+        else
+          unless @fields[name].kind_of?(RecordSchema)
+            raise ConfigError, "field #{name} is required to be a record but already registered as #{@field[name]}"
+          end
+        end
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-23 00:00:00.000000000 Z
+date: 2014-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake