fluent-plugin-bigquery-test 2.2.0

data/Rakefile

@@ -0,0 +1,12 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.warning = false
+end
+
+task :default => :test

data/fluent-plugin-bigquery.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent/plugin/bigquery/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-bigquery-test"
+  spec.version       = Fluent::BigQueryPlugin::VERSION
+  spec.authors       = ["dhayakawa"]
+  spec.description   = %q{Fluentd plugin to store data on Google BigQuery, by load, or by stream inserts}
+  spec.summary       = %q{Fluentd plugin to store data on Google BigQuery}
+  spec.homepage      = "https://github.com/kaizenplatform/fluent-plugin-bigquery"
+  spec.license       = "Apache-2.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rr"
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "test-unit-rr"
+
+  spec.add_runtime_dependency "google-api-client", ">= 0.49.0"
+  spec.add_runtime_dependency "googleauth", ">= 0.5.0"
+  spec.add_runtime_dependency "multi_json"
+  spec.add_runtime_dependency "fluentd", ">= 0.14.0", "< 2"
+end

data/gemfiles/activesupport-4.gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gem 'activesupport', '< 5'
+
+# Specify your gem's dependencies in fluent-plugin-bigquery.gemspec
+gemspec :path => '../'

data/lib/fluent/plugin/bigquery/errors.rb

@@ -0,0 +1,84 @@
+module Fluent
+  module BigQuery
+    # @abstract
+    class Error < StandardError
+      RETRYABLE_ERROR_REASON = %w(backendError internalError rateLimitExceeded tableUnavailable).freeze
+      RETRYABLE_INSERT_ERRORS_REASON = %w(timeout backendError internalError rateLimitExceeded).freeze
+      RETRYABLE_STATUS_CODE = [500, 502, 503, 504]
+
+      class << self
+        # @param e [Google::Apis::Error]
+        # @param message [String]
+        def wrap(e, message = nil)
+          if retryable_error?(e)
+            RetryableError.new(message, e)
+          else
+            UnRetryableError.new(message, e)
+          end
+        end
+
+        # @param e [Google::Apis::Error]
+        def retryable_error?(e)
+          e.is_a?(Google::Apis::ServerError) && RETRYABLE_STATUS_CODE.include?(e.status_code)
+        end
+
+        def retryable_error_reason?(reason)
+          RETRYABLE_ERROR_REASON.include?(reason)
+        end
+
+        def retryable_insert_errors_reason?(reason)
+          RETRYABLE_INSERT_ERRORS_REASON.include?(reason)
+        end
+
+        # Guard for instantiation
+        private :new
+        def inherited(subclass)
+          subclass.class_eval do
+            class << self
+              public :new
+            end
+          end
+        end
+      end
+
+      attr_reader :origin
+
+      def initialize(message, origin = nil)
+        @origin = origin
+        super(message || origin.message)
+      end
+
+      def method_missing(name, *args)
+        if @origin
+          @origin.send(name, *args)
+        else
+          super
+        end
+      end
+
+      def reason
+        @origin && @origin.respond_to?(:reason) ? @origin.reason : nil
+      end
+
+      def status_code
+        @origin && @origin.respond_to?(:status_code) ? @origin.status_code : nil
+      end
+
+      def body
+        @origin && @origin.respond_to?(:body) ? @origin.body : nil
+      end
+
+      def retryable?
+        false
+      end
+    end
+
+    class UnRetryableError < Error; end
+
+    class RetryableError < Error
+      def retryable?
+        true
+      end
+    end
+  end
+end

data/lib/fluent/plugin/bigquery/helper.rb

@@ -0,0 +1,33 @@
+module Fluent
+  module BigQuery
+    module Helper
+      class << self
+        def deep_symbolize_keys(object)
+          case object
+          when Hash
+            object.each_with_object({}) do |(key, value), result|
+              result[key.to_sym] = deep_symbolize_keys(value)
+            end
+          when Array
+            object.map {|e| deep_symbolize_keys(e) }
+          else
+            object
+          end
+        end
+
+        def deep_stringify_keys(object)
+          case object
+          when Hash
+            object.each_with_object({}) do |(key, value), result|
+              result[key.to_s] = deep_stringify_keys(value)
+            end
+          when Array
+            object.map {|e| deep_stringify_keys(e) }
+          else
+            object
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/bigquery/schema.rb

@@ -0,0 +1,281 @@
+require 'multi_json'
+
+module Fluent
+  module BigQuery
+    class FieldSchema
+      def initialize(name, mode = :nullable)
+        unless [:nullable, :required, :repeated].include?(mode)
+          raise ConfigError, "Unrecognized mode for #{name}: #{mode}"
+        end
+        ### https://developers.google.com/bigquery/docs/tables
+        # Each field has the following properties:
+        #
+        # name - The name must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_),
+        #        and must start with a letter or underscore. The maximum length is 128 characters.
+        #        https://cloud.google.com/bigquery/docs/reference/v2/tables#schema.fields.name
+        unless name =~ /^[_A-Za-z][_A-Za-z0-9]{,127}$/
+          raise ConfigError, "invalid bigquery field name: '#{name}'"
+        end
+
+        @name = name
+        @mode = mode
+      end
+
+      attr_reader :name, :mode
+
+      def format(value)
+        case @mode
+        when :nullable
+          format_one(value) unless value.nil?
+        when :required
+          if value.nil?
+            log.warn "Required field #{name} cannot be null"
+            nil
+          else
+            format_one(value)
+          end
+        when :repeated
+          value.nil? ? [] : value.each_with_object([]) { |v, arr| arr << format_one(v) if v }
+        end
+      end
+
+      def format_one(value)
+        raise NotImplementedError, "Must implement in a subclass"
+      end
+
+      def to_h
+        {
+          :name => name,
+          :type => type.to_s.upcase,
+          :mode => mode.to_s.upcase,
+        }
+      end
+    end
+
+    class StringFieldSchema < FieldSchema
+      def type
+        :string
+      end
+
+      def format_one(value)
+        if value.is_a?(Hash) || value.is_a?(Array)
+          MultiJson.dump(value)
+        else
+          value.to_s
+        end
+      end
+    end
+
+    class IntegerFieldSchema < FieldSchema
+      def type
+        :integer
+      end
+
+      def format_one(value)
+        value.to_i
+      end
+    end
+
+    class FloatFieldSchema < FieldSchema
+      def type
+        :float
+      end
+
+      def format_one(value)
+        value.to_f
+      end
+    end
+
+    class NumericFieldSchema < FieldSchema
+      def type
+        :numeric
+      end
+
+      def format_one(value)
+        value.to_s
+      end
+    end
+
+    class BooleanFieldSchema < FieldSchema
+      def type
+        :boolean
+      end
+
+      def format_one(value)
+        !!value
+      end
+    end
+
+    class TimestampFieldSchema < FieldSchema
+      INTEGER_REGEXP = /\A-?[[:digit:]]+\z/.freeze
+      FLOAT_REGEXP = /\A-?[[:digit:]]+(\.[[:digit:]]+)\z/.freeze
+
+      def type
+        :timestamp
+      end
+
+      def format_one(value)
+        case value
+        when Time
+          value.strftime("%Y-%m-%d %H:%M:%S.%6L %:z")
+        when String
+          if value =~ INTEGER_REGEXP
+            value.to_i
+          elsif value =~ FLOAT_REGEXP
+            value.to_f
+          else
+            value
+          end
+        else
+          value
+        end
+      end
+    end
+
+    class DateFieldSchema < FieldSchema
+      def type
+        :date
+      end
+
+      def format_one(value)
+        if value.respond_to?(:strftime)
+          value.strftime("%Y-%m-%d")
+        else
+          value
+        end
+      end
+    end
+
+    class DateTimeFieldSchema < FieldSchema
+      def type
+        :datetime
+      end
+
+      def format_one(value)
+        if value.respond_to?(:strftime)
+          value.strftime("%Y-%m-%dT%H:%M:%S.%6L")
+        else
+          value
+        end
+      end
+    end
+
+    class TimeFieldSchema < FieldSchema
+      def type
+        :time
+      end
+
+      def format_one(value)
+        if value.respond_to?(:strftime)
+          value.strftime("%H:%M:%S.%6L")
+        else
+          value
+        end
+      end
+    end
+
+    class RecordSchema < FieldSchema
+      FIELD_TYPES = {
+        string: StringFieldSchema,
+        integer: IntegerFieldSchema,
+        float: FloatFieldSchema,
+        numeric: NumericFieldSchema,
+        boolean: BooleanFieldSchema,
+        timestamp: TimestampFieldSchema,
+        date: DateFieldSchema,
+        datetime: DateTimeFieldSchema,
+        time: TimeFieldSchema,
+        record: RecordSchema
+      }.freeze
+
+      def initialize(name, mode = :nullable)
+        super(name, mode)
+        @fields = {}
+      end
+
+      def type
+        :record
+      end
+
+      def [](name)
+        @fields[name]
+      end
+
+      def empty?
+        @fields.empty?
+      end
+
+      def to_a
+        @fields.map do |_, field_schema|
+          field_schema.to_h
+        end
+      end
+
+      def to_h
+        {
+          :name => name,
+          :type => type.to_s.upcase,
+          :mode => mode.to_s.upcase,
+          :fields => self.to_a,
+        }
+      end
+
+      def load_schema(schema)
+        schema.each do |field|
+          raise ConfigError, 'field must have type' unless field.key?('type')
+
+          name = field['name']
+          mode = (field['mode'] || 'nullable').downcase.to_sym
+
+          type = field['type'].downcase.to_sym
+          field_schema_class = FIELD_TYPES[type]
+          raise ConfigError, "Invalid field type: #{field['type']}" unless field_schema_class
+
+          field_schema = field_schema_class.new(name, mode)
+          @fields[name] = field_schema
+          if type == :record
+            raise ConfigError, "record field must have fields" unless field.key?('fields')
+            field_schema.load_schema(field['fields'])
+          end
+        end
+      end
+
+      def register_field(name, type)
+        if @fields.key?(name) and @fields[name].type != :timestamp
+          raise ConfigError, "field #{name} is registered twice"
+        end
+        if name[/\./]
+          recordname = $`
+          fieldname = $'
+          register_record_field(recordname)
+          @fields[recordname].register_field(fieldname, type)
+        else
+          schema = FIELD_TYPES[type]
+          raise ConfigError, "[Bug] Invalid field type #{type}" unless schema
+          @fields[name] = schema.new(name)
+        end
+      end
+
+      def format_one(record)
+        out = {}
+        record.each do |key, value|
+          next if value.nil?
+          schema = @fields[key]
+          out[key] = schema ? schema.format(value) : value
+        end
+        out
+      end
+
+      private
+      def register_record_field(name)
+        if !@fields.key?(name)
+          @fields[name] = RecordSchema.new(name)
+        else
+          unless @fields[name].kind_of?(RecordSchema)
+            raise ConfigError, "field #{name} is required to be a record but already registered as #{@field[name]}"
+          end
+        end
+      end
+    end
+  end
+end