fluent-plugin-bigquery 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bc7d16f7400a52c0cbf655c22b6c919572142182
-  data.tar.gz: 7c83037e999617f73a0fc6a51d41167a979b72a9
+  metadata.gz: d6c2bb5c4890390bcf4d60938aef91661634dfb2
+  data.tar.gz: 37656e71ecc84fca33a84f3df82261b5ef7a29ae
 SHA512:
-  metadata.gz: a25328702bfad9ddff282cbcd9c8d84d64e2857158c1bc7a20ce388610bb5150e969ac0f07ebd1162b01ffca29b50cf920a28c3806715512cad4a9ff6cbe4d5c
-  data.tar.gz: 9701a7f10f17e0f7dbaf61bca99d6f73e1b769deba6361935cc413c1bbc5039349c626a84309f57919e884228cc3084eb38d1fdad83fde99505a68509d171145
+  metadata.gz: cdb2f47e86521fb3484335dcb076a29be9004f70fff3a2105735b9876eb0ace6b832d2ff8891a926f087f32ad20fd41aa55b441b89ce31c99c6c42b8a4474d87
+  data.tar.gz: 0dedda63c139ff185f7c75cc3d542b2804784f8fefada2956d67b8157e784725409aa5b19bc4ac85fb3bab8e00c9e855680ed124a189dbe7ed224671983230fc

data/lib/fluent/plugin/bigquery/version.rb

@@ -1,6 +1,6 @@
 module Fluent
   module BigQueryPlugin
-    VERSION = "0.2.0"
+    VERSION = "0.2.1"
   end
 end
 

data/lib/fluent/plugin/out_bigquery.rb

@@ -311,7 +311,8 @@ module Fluent
     end
 
     def fetch_schema
-      table_id = @tablelist[0]
+      table_id_format = @tablelist[0]
+      table_id = generate_table_id(table_id_format, Time.at(Fluent::Engine.now))
       res = client.execute(
         :api_method => @bq.tables.get,
         :parameters => {

data/test/plugin/test_out_bigquery.rb

@@ -458,39 +458,7 @@ class BigQueryOutputTest < Test::Unit::TestCase
       ) {
         s = stub!
         s.success? { true }
-        s.body {
-          JSON.generate({
-            schema: {
-              fields: [
-                {
-                  name: "time",
-                  type: "TIMESTAMP",
-                  mode: "REQUIRED"
-                },
-                {
-                  name: "tty",
-                  type: "STRING",
-                  mode: "NULLABLE"
-                },
-                {
-                  name: "pwd",
-                  type: "STRING",
-                  mode: "REQUIRED"
-                },
-                {
-                  name: "user",
-                  type: "STRING",
-                  mode: "REQUIRED"
-                },
-                {
-                  name: "argv",
-                  type: "STRING",
-                  mode: "REPEATED"
-                }
-              ]
-            }
-          })
-        }
+        s.body { JSON.generate(sudo_schema_response) }
         s
       }
     end
@@ -498,11 +466,90 @@ class BigQueryOutputTest < Test::Unit::TestCase
     buf = driver.instance.format_stream("my.tag", [input])
     driver.instance.shutdown
 
+    assert_equal expected, MessagePack.unpack(buf)
+
     fields = driver.instance.instance_eval{ @fields }
     assert fields["time"]
     assert_equal :integer, fields["time"].type  # DO NOT OVERWRITE
     assert_equal :nullable, fields["time"].mode # DO NOT OVERWRITE
+
+    assert fields["tty"]
+    assert_equal :string, fields["tty"].type
+    assert_equal :nullable, fields["tty"].mode
     
+    assert fields["pwd"]
+    assert_equal :string, fields["pwd"].type
+    assert_equal :required, fields["pwd"].mode
+
+    assert fields["user"]
+    assert_equal :string, fields["user"].type
+    assert_equal :required, fields["user"].mode
+
+    assert fields["argv"]
+    assert_equal :string, fields["argv"].type
+    assert_equal :repeated, fields["argv"].mode
+  end
+
+  def test_format_fetch_from_bigquery_api_with_generated_table_id
+    now = Time.now
+    input = [
+      now,
+      {
+        "tty" => nil,
+        "pwd" => "/home/yugui",
+        "user" => "fluentd",
+        "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
+      }
+    ]
+    expected = {
+      "json" => {
+        "time" => now.to_i,
+        "pwd" => "/home/yugui",
+        "user" => "fluentd",
+        "argv" => %w[ tail -f /var/log/fluentd/fluentd.log ]
+      }
+    }
+
+    driver = create_driver(<<-CONFIG)
+      table foo_%Y_%m_%d
+      email foo@bar.example
+      private_key_path /path/to/key
+      project yourproject_id
+      dataset yourdataset_id
+
+      time_format %s
+      time_field  time
+
+      fetch_schema true
+      field_integer time
+    CONFIG
+    mock_client(driver) do |expect|
+      expect.discovered_api("bigquery", "v2") { mock!.tables.mock!.get { Object.new } }
+      expect.execute(
+        :api_method => anything,
+        :parameters => {
+          'projectId' => 'yourproject_id',
+          'datasetId' => 'yourdataset_id',
+          'tableId' => now.strftime('foo_%Y_%m_%d')
+        }
+      ) {
+        s = stub!
+        s.success? { true }
+        s.body { JSON.generate(sudo_schema_response) }
+        s
+      }
+    end
+    driver.instance.start
+    buf = driver.instance.format_stream("my.tag", [input])
+    driver.instance.shutdown
+
+    assert_equal expected, MessagePack.unpack(buf)
+
+    fields = driver.instance.instance_eval{ @fields }
+    assert fields["time"]
+    assert_equal :integer, fields["time"].type  # DO NOT OVERWRITE
+    assert_equal :nullable, fields["time"].mode # DO NOT OVERWRITE
+
     assert fields["tty"]
     assert_equal :string, fields["tty"].type
     assert_equal :nullable, fields["tty"].mode
@@ -588,4 +635,40 @@ class BigQueryOutputTest < Test::Unit::TestCase
     table_id = driver.instance.generate_table_id(table_id_format, time)
     assert_equal 'foo_2014_08_11', table_id
   end
+
+  private
+
+  def sudo_schema_response
+    {
+      schema: {
+        fields: [
+          {
+            name: "time",
+            type: "TIMESTAMP",
+            mode: "REQUIRED"
+          },
+          {
+            name: "tty",
+            type: "STRING",
+            mode: "NULLABLE"
+          },
+          {
+            name: "pwd",
+            type: "STRING",
+            mode: "REQUIRED"
+          },
+          {
+            name: "user",
+            type: "STRING",
+            mode: "REQUIRED"
+          },
+          {
+            name: "argv",
+            type: "STRING",
+            mode: "REPEATED"
+          }
+        ]
+      }
+    }
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Naoya Ito