embulk-output-bigquery 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7e6de19fdc976904bd2ac46aa529e030f9ecbe48
-  data.tar.gz: d279269a8914553203a1d2bc06832ed4201bedf3
+  metadata.gz: daa84fac7fd9adf3ad62798ef5b113ed462fc303
+  data.tar.gz: fafbd5023df0b9d3539a754b699e089012d4caa1
 SHA512:
-  metadata.gz: 3e7f86e4b963012e0a55227b53262c141209d5aaed5c06b82d1d3da844f467c2b0e53c750e31eec38963459e58e02ba54b82fbb39c14084df241428b5245a220
-  data.tar.gz: 60850b5ce34b7b626cf9766a537196d160ae1c1bd0365f119817f50977f7a171762e6562ac987cdc2655fc75892dd6beafceebd47305818e625330aaabb1911b
+  metadata.gz: 5a273f26c8c357b082d94bc97364f18fc73db9f829b12d5bd10df867f625d4619da59c483d56c83cbbdef82383e3a4116f185f9973c246cc4ccd8adc187cadb2
+  data.tar.gz: 6bd617783792cd5c4e7d3ab7e0b39ab3f86fa97cb326f3b6978e743e74902130c809f353bc3238f97ab5e6b439ba44cf9972dc2840d37b2b3ad5d8b94f6dfddd

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.2 - 2016-02-15
+
+* [new feature] Added template_table option. [#25](https://github.com/embulk/embulk-output-bigquery/pull/25)
+
 ## 0.2.1 - 2016-01-28
 
 * [maintenance] Upgraded Embulk version to 0.8.1 [#22](https://github.com/embulk/embulk-output-bigquery/pull/22). @joker1007 thanks!

data/README.md

@@ -40,6 +40,7 @@ OAuth flow for installed applications.
 |  table                    | string      | required   |                | table name |
 |  auto_create_table        | boolean     | optional   | 0              | [See below](#dynamic-table-creating) |
 |  schema_file              | string      | optional   |                | /path/to/schema.json |
+|  template_table           | string      | optional   |                | existing_table_name [See below](#dynamic-table-creating) |
 |  prevent_duplicate_insert | boolean     | optional   | 0              | [See below](#data-consistency) |
 |  delete_from_local_when_job_end | boolean     | optional   | 0            | If set to true, delete local file when job is end |
 |  job_status_max_polling_time    | int         | optional   | 3600 sec     | Max job status polling time |
@@ -195,8 +196,11 @@ When `auto_create_table` is set to true, try to create the table using BigQuery
 
 If table already exists, insert into it.
 
-To describe the schema of the target table, please write schema path.
+There are 2 ways to set schema.
 
+#### Set schema.json
+
+Please set file path of schema.json.
 
 ```yaml
 out:
@@ -206,6 +210,18 @@ out:
   schema_file: /path/to/schema.json
 ```
 
+#### Set template_table in dataset
+
+Plugin will try to read schema from existing table and use it as schema template.
+
+```yaml
+out:
+  type: bigquery
+  auto_create_table: true
+  table: table_%Y_%m
+  template_table: existing_table_name
+```
+
 ### Data Consistency
 
 When `prevent_duplicate_insert` is set to true, embulk-output-bigquery generate job ID from md5 hash of file  and other options to prevent duplicate data insertion.

data/build.gradle

@@ -16,7 +16,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.2.1"
+version = "0.2.2"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.1"

data/src/main/java/org/embulk/output/BigqueryAuthentication.java

@@ -1,27 +1,28 @@
 package org.embulk.output;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Collections;
-import java.security.GeneralSecurityException;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.api.client.googleapis.compute.ComputeCredential;
 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.client.http.HttpRequestInitializer;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
 import com.google.api.services.bigquery.Bigquery;
 import com.google.api.services.bigquery.BigqueryScopes;
 import com.google.api.services.bigquery.model.ProjectList;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
 
+import java.io.File;
+import java.io.FileInputStream;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.Collections;
+
 public class BigqueryAuthentication
 {
     private final Logger log = Exec.getLogger(BigqueryAuthentication.class);

data/src/main/java/org/embulk/output/BigqueryOutputPlugin.java

@@ -1,36 +1,36 @@
 package org.embulk.output;
 
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.security.NoSuchAlgorithmException;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
 import com.google.common.base.Function;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
-import java.security.GeneralSecurityException;
-import org.jruby.embed.ScriptingContainer;
-
 import org.embulk.config.Config;
-import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskReport;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
-import org.embulk.spi.unit.LocalFile;
 import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
 import org.embulk.spi.FileOutputPlugin;
 import org.embulk.spi.TransactionalFileOutput;
-import org.embulk.spi.Exec;
-
+import org.embulk.spi.unit.LocalFile;
+import org.jruby.embed.ScriptingContainer;
 import org.slf4j.Logger;
 
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.security.GeneralSecurityException;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
 public class BigqueryOutputPlugin
         implements FileOutputPlugin
 {
@@ -116,6 +116,10 @@ public class BigqueryOutputPlugin
         Optional<LocalFile> getSchemaFile();
         void setSchemaFile(Optional<LocalFile> schemaFile);
 
+        @Config("template_table")
+        @ConfigDefault("null")
+        Optional<String> getTemplateTable();
+
         @Config("prevent_duplicate_insert")
         @ConfigDefault("false")
         boolean getPreventDuplicateInsert();
@@ -209,8 +213,12 @@ public class BigqueryOutputPlugin
                     task.getP12Keyfile().transform(localFileToPathString()),
                     task.getJsonKeyfile().transform(localFileToPathString()),
                     task.getApplicationName())
+                    .setProject(task.getProject())
+                    .setDataset(task.getDataset())
+                    .setTable(task.getTable())
                     .setAutoCreateTable(task.getAutoCreateTable())
                     .setSchemaPath(task.getSchemaFile().transform(localFileToPathString()))
+                    .setTemplateTable(task.getTemplateTable())
                     .setSourceFormat(task.getSourceFormat().getString())
                     .setFieldDelimiter(String.valueOf(task.getFieldDelimiter()))
                     .setMaxBadRecords(task.getMaxBadrecords())
@@ -222,9 +230,6 @@ public class BigqueryOutputPlugin
                     .setIgnoreUnknownValues(task.getIgnoreUnknownValues())
                     .setAllowQuotedNewlines(task.getAllowQuotedNewlines())
                     .build();
-
-            bigQueryWriter.checkConfig(task.getProject(), task.getDataset(), task.getTable());
-
         }
         catch (IOException | GeneralSecurityException ex) {
             throw new ConfigException(ex);

data/src/main/java/org/embulk/output/BigqueryWriter.java

@@ -1,40 +1,52 @@
 package org.embulk.output;
 
-import java.io.File;
-import java.io.IOException;
-import java.io.FileNotFoundException;
-import java.io.FileInputStream;
-import java.io.BufferedInputStream;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.client.googleapis.media.MediaHttpUploader;
+import com.google.api.client.googleapis.media.MediaHttpUploaderProgressListener;
 import com.google.api.client.http.InputStreamContent;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-
-import com.google.api.services.bigquery.model.*;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.Bigquery.Jobs.Insert;
+import com.google.api.services.bigquery.Bigquery.Tables;
+import com.google.api.services.bigquery.model.ErrorProto;
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobConfiguration;
+import com.google.api.services.bigquery.model.JobConfigurationLoad;
+import com.google.api.services.bigquery.model.JobConfigurationTableCopy;
+import com.google.api.services.bigquery.model.JobReference;
+import com.google.api.services.bigquery.model.JobStatistics;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableFieldSchema;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
 import com.google.common.base.Optional;
-import java.security.GeneralSecurityException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.core.type.TypeReference;
-
 import com.google.common.collect.ImmutableList;
 import org.apache.commons.codec.binary.Hex;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
 
-import com.google.api.services.bigquery.Bigquery;
-import com.google.api.services.bigquery.Bigquery.Tables;
-import com.google.api.services.bigquery.Bigquery.Jobs.Insert;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.client.googleapis.media.MediaHttpUploader;
-import com.google.api.client.googleapis.media.MediaHttpUploaderProgressListener;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
 
 public class BigqueryWriter
 {
     private final Logger log = Exec.getLogger(BigqueryWriter.class);
+    private final String project;
+    private final String dataset;
+    private final String table;
     private final boolean autoCreateTable;
     private final Optional<String> schemaPath;
+    private final Optional<String> templateTable;
     private final TableSchema tableSchema;
     private final String sourceFormat;
     private final String fieldDelimiter;
@@ -51,8 +63,12 @@ public class BigqueryWriter
     public BigqueryWriter(Builder builder)
             throws IOException, GeneralSecurityException
     {
+        this.project = builder.project;
+        this.dataset = builder.dataset;
+        this.table = builder.table;
         this.autoCreateTable = builder.autoCreateTable;
         this.schemaPath = builder.schemaPath;
+        this.templateTable = builder.templateTable;
         this.sourceFormat = builder.sourceFormat.toUpperCase();
         this.fieldDelimiter = builder.fieldDelimiter;
         this.maxBadRecords = builder.maxBadRecords;
@@ -70,8 +86,15 @@ public class BigqueryWriter
         );
         this.bigQueryClient = auth.getBigqueryClient();
 
+        checkConfig();
+
         if (autoCreateTable) {
-            this.tableSchema = createTableSchema();
+            if (schemaPath.isPresent()) {
+                this.tableSchema = createTableSchema();
+            }
+            else {
+               this.tableSchema = fetchTableSchema();
+            }
         }
         else {
             this.tableSchema = null;
@@ -314,6 +337,15 @@ public class BigqueryWriter
         }
     }
 
+    public TableSchema fetchTableSchema() throws IOException
+    {
+        String fetchTarget = templateTable.orNull();
+        log.info(String.format("Fetch table schema from project:%s dataset:%s table:%s", project, dataset, fetchTarget));
+        Tables tableRequest = bigQueryClient.tables();
+        Table tableData = tableRequest.get(project, dataset, fetchTarget).execute();
+        return tableData.getSchema();
+    }
+
     public boolean isExistTable(String project, String dataset, String table) throws IOException
     {
         Tables tableRequest = bigQueryClient.tables();
@@ -326,18 +358,18 @@ public class BigqueryWriter
         return true;
     }
 
-    public void checkConfig(String project, String dataset, String table) throws IOException
+    public void checkConfig() throws IOException
     {
         if (autoCreateTable) {
-            if (!schemaPath.isPresent()) {
-                throw new FileNotFoundException("schema_file is empty");
-            }
-            else {
+            if (schemaPath.isPresent()) {
                 File file = new File(schemaPath.orNull());
                 if (!file.exists()) {
                     throw new FileNotFoundException("Can not load schema file.");
                 }
             }
+            else if (!templateTable.isPresent()) {
+                throw new FileNotFoundException("schema_file or template_table must be present");
+            }
         }
         else {
             if (!isExistTable(project, dataset, table)) {
@@ -404,8 +436,12 @@ public class BigqueryWriter
         private Optional<String> p12KeyFilePath;
         private Optional<String> jsonKeyFilePath;
         private String applicationName;
+        private String project;
+        private String dataset;
+        private String table;
         private boolean autoCreateTable;
         private Optional<String> schemaPath;
+        private Optional<String> templateTable;
         private String sourceFormat;
         private String fieldDelimiter;
         private int maxBadRecords;
@@ -427,6 +463,24 @@ public class BigqueryWriter
             this.applicationName = applicationName;
         }
 
+        public Builder setProject(String project)
+        {
+            this.project = project;
+            return this;
+        }
+
+        public Builder setDataset(String dataset)
+        {
+            this.dataset = dataset;
+            return this;
+        }
+
+        public Builder setTable(String table)
+        {
+            this.table = table;
+            return this;
+        }
+
         public Builder setAutoCreateTable(boolean autoCreateTable)
         {
             this.autoCreateTable = autoCreateTable;
@@ -439,6 +493,12 @@ public class BigqueryWriter
             return this;
         }
 
+        public Builder setTemplateTable(Optional<String> templateTable)
+        {
+            this.templateTable = templateTable;
+            return this;
+        }
+
         public Builder setSourceFormat(String sourceFormat)
         {
             this.sourceFormat = sourceFormat;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-28 00:00:00.000000000 Z
+date: 2016-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,7 +65,7 @@ files:
 - src/test/java/org/embulk/output/TestBigqueryWriter.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-output-bigquery-0.2.1.jar
+- classpath/embulk-output-bigquery-0.2.2.jar
 - classpath/google-api-client-1.20.0.jar
 - classpath/google-api-services-bigquery-v2-rev205-1.20.0.jar
 - classpath/google-http-client-1.20.0.jar