fluent-plugin-bigquery 0.5.0.beta2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 023c14a3ce8139a7b7cc9670ba7e5046e9d7407b
-  data.tar.gz: 7d5988af8f30c05149cb1f92770f72ab677074ce
+  metadata.gz: 570345f8dc6c8f21186e633cb5be03d40ed3480d
+  data.tar.gz: 26e474f9c08a3d8b7df2cd5de3a4121d37847f95
 SHA512:
-  metadata.gz: 48327b0270d7fe66913fbabd04b6e09a3270808ff28d83cfcfde30f8985b333f493e007350d3f0cca7d495bd9d57856ad48f974f851781eed66cdb40b0f705f5
-  data.tar.gz: 8522ffdfc297465313938ee2c9d6fff5979c390b5c82e70e192774136288df19111db24b42d08654505e84e920776c2db4b1967bd989f2539d93a7b2741f58b7
+  metadata.gz: dc887f24c5d975cea0c07f4bc1ede48755b396d1bc174c1ade842c6fe6e5d7cd729b75a9ab4bebaf2fc96aa5895f15aed41851a3eb11256ae681e2e347c073f3
+  data.tar.gz: b7e6fe7d2eb2562264b195cca7bc9f6f605dfeea8dd46c73cea8261e399e994f6d181cb0ac5a352fcdc92e665be9baca297b65861083dec2743f90118aeb42c9

data/.gitignore

@@ -17,3 +17,5 @@ test/tmp
 test/version_tmp
 tmp
 script/
+
+fluentd-0.12

data/.travis.yml

@@ -19,7 +19,4 @@ matrix:
 before_install:
   - gem update bundler
 
-before_install:
-  - gem update bundler
-
 script: bundle exec rake test

data/README.md

@@ -2,7 +2,7 @@
 
 [Fluentd](http://fluentd.org) output plugin to load/insert data into Google BigQuery.
 
-- **Plugin type**: TimeSlicedOutput
+- **Plugin type**: BufferedOutput
 
 * insert data over streaming inserts
   * for continuous real-time insertions
@@ -14,8 +14,11 @@
 Current version of this plugin supports Google API with Service Account Authentication, but does not support
 OAuth flow for installed applications.
 
-## Notice
-If you use ruby-2.1 or earlier, you must use activesupport-4.2.x or earlier.
+## Version Information
+v1.0.0 or later supports fluentd-0.14.0 or later.
+If you use fluentd-0.12.x, please use v0.4.x.
+
+I recommend to update fluentd version to v0.14.x or later.
 
 ## With docker image
 If you use official alpine based fluentd docker image (https://github.com/fluent/fluentd-docker-image),

data/fluent-plugin-bigquery.gemspec

@@ -23,9 +23,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit"
   spec.add_development_dependency "test-unit-rr"
 
-  spec.add_runtime_dependency "google-api-client", "~> 0.9.3"
+  spec.add_runtime_dependency "google-api-client", ">= 0.9.3", "< 0.14"
   spec.add_runtime_dependency "googleauth", ">= 0.5.0"
   spec.add_runtime_dependency "multi_json"
-  spec.add_runtime_dependency "activesupport", ">= 3.2", "< 6"
   spec.add_runtime_dependency "fluentd", "~> 0.14.0"
 end

data/lib/fluent/plugin/bigquery/helper.rb

@@ -0,0 +1,33 @@
+module Fluent
+  module BigQuery
+    module Helper
+      class << self
+        def deep_symbolize_keys(object)
+          case object
+          when Hash
+            object.each_with_object({}) do |(key, value), result|
+              result[key.to_sym] = deep_symbolize_keys(value)
+            end
+          when Array
+            object.map {|e| deep_symbolize_keys(e) }
+          else
+            object
+          end
+        end
+
+        def deep_stringify_keys(object)
+          case object
+          when Hash
+            object.each_with_object({}) do |(key, value), result|
+              result[key.to_s] = deep_stringify_keys(value)
+            end
+          when Array
+            object.map {|e| deep_stringify_keys(e) }
+          else
+            object
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,5 +1,5 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.5.0.beta2".freeze
+    VERSION = "1.0.0".freeze
   end
 end

data/lib/fluent/plugin/bigquery/writer.rb

@@ -42,7 +42,7 @@ module Fluent
             definition[:time_partitioning] = {
               type: @options[:time_partitioning_type].to_s.upcase,
               expiration_ms: @options[:time_partitioning_expiration] ? @options[:time_partitioning_expiration] * 1000 : nil
-            }.compact
+            }.select { |_, value| !value.nil? }
           end
           client.insert_table(project, dataset, definition, {})
           log.debug "create table", project_id: project, dataset: dataset, table: table_id
@@ -73,7 +73,7 @@ module Fluent
 
       def fetch_schema(project, dataset, table_id)
         res = client.get_table(project, dataset, table_id)
-        schema = res.schema.fields.as_json
+        schema = Fluent::BigQuery::Helper.deep_stringify_keys(res.schema.to_h[:fields])
         log.debug "Load schema from BigQuery: #{project}:#{dataset}.#{table_id} #{schema}"
 
         schema

data/lib/fluent/plugin/out_bigquery.rb

@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
 
+require 'fluent/plugin/output'
+
 require 'fluent/plugin/bigquery/version'
 
+require 'fluent/plugin/bigquery/helper'
 require 'fluent/plugin/bigquery/errors'
 require 'fluent/plugin/bigquery/schema'
 require 'fluent/plugin/bigquery/writer'
@@ -174,9 +177,6 @@ module Fluent
         require 'multi_json'
         require 'google/apis/bigquery_v2'
         require 'googleauth'
-        require 'active_support/json'
-        require 'active_support/core_ext/hash'
-        require 'active_support/core_ext/object/json'
 
         # MEMO: signet-0.6.1 depend on Farady.default_connection
         Faraday.default_connection.options.timeout = 60
@@ -411,7 +411,7 @@ module Fluent
               record[@add_insert_timestamp] = now if @add_insert_timestamp
               row = {"json" => record}
               row["insert_id"] = @get_insert_id.call(record) if @get_insert_id
-              row.deep_symbolize_keys
+              Fluent::BigQuery::Helper.deep_symbolize_keys(row)
             end
           end
 

data/test/helper.rb

@@ -1,25 +1,9 @@
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
+require 'bundler/setup'
 require 'test/unit'
 
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(__dir__)
 require 'fluent/test'
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
 
 require 'fluent/plugin/buffer'
 require 'fluent/plugin/buf_memory'
@@ -31,8 +15,4 @@ require 'google/apis/bigquery_v2'
 require 'google/api_client/auth/key_utils'
 require 'googleauth'
 
-require 'rr'
 require 'test/unit/rr'
-
-class Test::Unit::TestCase
-end

data/test/plugin/test_out_bigquery.rb

@@ -481,7 +481,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
 
     writer = stub_writer(driver)
     mock(writer).fetch_schema('yourproject_id', 'yourdataset_id', 'foo') do
-      sudo_schema_response.deep_stringify_keys["schema"]["fields"]
+      sudo_schema_response["schema"]["fields"]
     end
 
     buf = nil
@@ -549,7 +549,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
 
     writer = stub_writer(driver)
     mock(writer).fetch_schema('yourproject_id', 'yourdataset_id', 'foo') do
-      sudo_schema_response.deep_stringify_keys["schema"]["fields"]
+      sudo_schema_response["schema"]["fields"]
     end
 
     buf = nil
@@ -913,7 +913,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
 
       buffer_type memory
     CONFIG
-    schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
+    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(schema_path)))
 
     writer = stub_writer(driver)
     io = StringIO.new("hello")
@@ -969,7 +969,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
 
       buffer_type memory
     CONFIG
-    schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
+    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(schema_path)))
 
     io = StringIO.new("hello")
     mock(driver.instance).create_upload_source(is_a(Fluent::Plugin::Buffer::Chunk)).yields(io)
@@ -1025,7 +1025,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
 
       buffer_type memory
     CONFIG
-    schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
+    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(schema_path)))
 
     driver.instance_start
     tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
@@ -1106,7 +1106,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
         utc
       </secondary>
     CONFIG
-    schema_fields = MultiJson.load(File.read(schema_path)).map(&:deep_symbolize_keys)
+    schema_fields = Fluent::BigQuery::Helper.deep_symbolize_keys(MultiJson.load(File.read(schema_path)))
 
     driver.instance_start
     tag, time, record = "tag", Time.now.to_i, {"a" => "b"}
@@ -1236,7 +1236,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
       schema_path #{File.join(File.dirname(__FILE__), "testdata", "apache.schema")}
     CONFIG
     writer = stub_writer(driver)
-    mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [{json: message.deep_symbolize_keys}], template_suffix: nil) do
+    mock(writer).insert_rows('yourproject_id', 'yourdataset_id', 'foo', [{json: Fluent::BigQuery::Helper.deep_symbolize_keys(message)}], template_suffix: nil) do
       raise Fluent::BigQuery::RetryableError.new(nil, Google::Apis::ServerError.new("Not found: Table yourproject_id:yourdataset_id.foo", status_code: 404, body: "Not found: Table yourproject_id:yourdataset_id.foo"))
     end
     mock(writer).create_table('yourproject_id', 'yourdataset_id', 'foo', driver.instance.instance_variable_get(:@table_schema))
@@ -1251,30 +1251,30 @@ class BigQueryOutputTest < Test::Unit::TestCase
   def test_auto_create_partitioned_table_by_bigquery_api
     now = Time.now
     message = {
-      "json" => {
-        "time" => now.to_i,
-        "request" => {
-          "vhost" => "bar",
-          "path" => "/path/to/baz",
-          "method" => "GET",
-          "protocol" => "HTTP/1.0",
-          "agent" => "libwww",
-          "referer" => "http://referer.example",
-          "time" => (now - 1).to_f,
-          "bot_access" => true,
-          "loginsession" => false,
+      json: {
+        time: now.to_i,
+        request: {
+          vhost: "bar",
+          path: "/path/to/baz",
+          method: "GET",
+          protocol: "HTTP/1.0",
+          agent: "libwww",
+          referer: "http://referer.example",
+          time: (now - 1).to_f,
+          bot_access: true,
+          loginsession: false,
         },
-        "remote" => {
-          "host" => "remote.example",
-          "ip" =>  "192.168.1.1",
-          "user" => "nagachika",
+        remote: {
+          host: "remote.example",
+          ip: "192.168.1.1",
+          user: "nagachika",
         },
-        "response" => {
-          "status" => 200,
-          "bytes" => 72,
+        response: {
+          status: 200,
+          bytes: 72,
         },
       }
-    }.deep_symbolize_keys
+    }
 
     driver = create_driver(<<-CONFIG)
       table foo
@@ -1309,32 +1309,32 @@ class BigQueryOutputTest < Test::Unit::TestCase
 
   def sudo_schema_response
     {
-      schema: {
-        fields: [
+      "schema" => {
+        "fields" => [
           {
-            name: "time",
-            type: "TIMESTAMP",
-            mode: "REQUIRED"
+            "name" => "time",
+            "type" => "TIMESTAMP",
+            "mode" => "REQUIRED"
           },
           {
-            name: "tty",
-            type: "STRING",
-            mode: "NULLABLE"
+            "name" => "tty",
+            "type" => "STRING",
+            "mode" => "NULLABLE"
           },
           {
-            name: "pwd",
-            type: "STRING",
-            mode: "REQUIRED"
+            "name" => "pwd",
+            "type" => "STRING",
+            "mode" => "REQUIRED"
           },
           {
-            name: "user",
-            type: "STRING",
-            mode: "REQUIRED"
+            "name" => "user",
+            "type" => "STRING",
+            "mode" => "REQUIRED"
           },
           {
-            name: "argv",
-            type: "STRING",
-            mode: "REPEATED"
+            "name" => "argv",
+            "type" => "STRING",
+            "mode" => "REPEATED"
           }
         ]
       }

data/test/plugin/test_record_schema.rb

@@ -1,7 +1,4 @@
 require 'helper'
-require 'active_support/json'
-require 'active_support/core_ext/hash'
-require 'active_support/core_ext/object/json'
 
 class RecordSchemaTest < Test::Unit::TestCase
   def base_schema
@@ -102,7 +99,7 @@ class RecordSchemaTest < Test::Unit::TestCase
   def test_load_schema
     fields = Fluent::BigQuery::RecordSchema.new("record")
     fields.load_schema(base_schema)
-    assert { fields.to_a.as_json == base_schema }
+    assert { Fluent::BigQuery::Helper.deep_stringify_keys(fields.to_a) == base_schema }
   end
 
   def test_load_schema_allow_overwrite_with_type_changed_column
@@ -110,7 +107,7 @@ class RecordSchemaTest < Test::Unit::TestCase
     fields.load_schema(base_schema)
 
     fields.load_schema(base_schema_with_type_changed_column)
-    assert { fields.to_a.as_json == base_schema_with_type_changed_column }
+    assert { Fluent::BigQuery::Helper.deep_stringify_keys(fields.to_a) == base_schema_with_type_changed_column }
   end
 
   def test_load_schema_allow_overwrite_with_new_column
@@ -118,7 +115,7 @@ class RecordSchemaTest < Test::Unit::TestCase
     fields.load_schema(base_schema)
 
     fields.load_schema(base_schema_with_new_column)
-    assert { fields.to_a.as_json == base_schema_with_new_column }
+    assert { Fluent::BigQuery::Helper.deep_stringify_keys(fields.to_a) == base_schema_with_new_column }
   end
 
   def test_format_one

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.5.0.beta2
+  version: 1.0.0
 platform: ruby
 authors:
 - Naoya Ito
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-18 00:00:00.000000000 Z
+date: 2017-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -71,16 +71,22 @@ dependencies:
   name: google-api-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: googleauth
   requirement: !ruby/object:Gem::Requirement
@@ -109,26 +115,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -161,6 +147,7 @@ files:
 - fluent-plugin-bigquery.gemspec
 - gemfiles/activesupport-4.gemfile
 - lib/fluent/plugin/bigquery/errors.rb
+- lib/fluent/plugin/bigquery/helper.rb
 - lib/fluent/plugin/bigquery/schema.rb
 - lib/fluent/plugin/bigquery/version.rb
 - lib/fluent/plugin/bigquery/writer.rb
@@ -187,9 +174,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.8