embulk-output-bigquery 0.1.11 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9eed778576242d0ea0d44e445c5f126e93cc6333
-  data.tar.gz: 96bc25e1ff2caccfbd7f2a4be1730b80dca185de
+  metadata.gz: 9e4b4a5467fb4589a3d88541af95ee5b002a5f9d
+  data.tar.gz: 2845c3d881b88183bdf3355647fbfecd6e455372
 SHA512:
-  metadata.gz: 95d91febb961ede313d9aa66e32a177812e3a32a96203de2d82cc8d5682517be5f688997b16123d0cedd820d858762fc90755dce109ce89f32b6bb96496226e7
-  data.tar.gz: 778932e6c37a30992ffb2593ee69cb6be463369d122b8339e794ec063083606dd486fbfe5b21f31a6468556e1166654a3caa22422bc0f78af75c2396291ddd08
+  metadata.gz: 61e5481dd18b750555f3462d680cb2b4d680ce768683780e4b2ce2cac6e66134c7eb38227adf8e48a4655447e27e166f14487c129fe9154a164131a4be06270b
+  data.tar.gz: 09036cb1d4b20556cb36e11839c7807b00ae453976b4192f9c6f489284aa5f1cbe48b6c42e5b9149b1ed3c3d04a92ee9546cce030b3d9e0c0929f70e365d3ae7

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.2.0 - 2016-01-26
+
+* Added mode parameters and support 4 modes(append, replace, replace_backup, delete_in_advance). @joker1007 thanks!
+
 Release 0.1.11 - 2015-11-16
 
 * Change error result display for easy investigation.

data/README.md

@@ -28,6 +28,7 @@ OAuth flow for installed applications.
 
 | name                      | type        | required?  | default      | description            |  
 |:--------------------------|:------------|:-----------|:-------------|:-----------------------|
+|  mode                     | string      | optional   | append       | [See below](#mode)     |
 |  auth_method              | string      | optional   | "private_key"  | `private_key` , `json_key` or `compute_engine`
 |  service_account_email    | string      | required when auth_method is private_key  |   | Your Google service account email
 |  p12_keyfile              | string      | required when auth_method is private_key   |   | Fullpath of private key in P12(PKCS12) format |
@@ -80,6 +81,36 @@ out:
   - {type: gzip}
 ```
 
+### mode
+
+4 modes are provided.
+
+#### append
+
+default. When append mode, plugin will insert data into existing table.
+
+#### replace
+
+1. Load to temporary table.
+2. Copy temporary table to destination table. (WRITE_TRUNCATE)
+
+```is_skip_job_result_check``` must be false when replace mode
+
+#### replace_backup
+
+1. Load to temporary table.
+2. Copy destination table to backup table. (table_name_old)
+3. Copy temporary table to destination table. (WRITE_TRUNCATE)
+
+```is_skip_job_result_check``` must be false when replace_backup mode.
+
+#### delete_in_advance
+
+1. Delete destination table, if it exists.
+2. Load to destination table.
+
+```auto_create_table``` must be true when delete_in_advance mode.
+
 ### Authentication
 
 There are three methods supported to fetch access token for the service account.

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.11"
+version = "0.2.0"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.1"

data/src/main/java/org/embulk/output/BigqueryOutputPlugin.java

@@ -139,9 +139,14 @@ public class BigqueryOutputPlugin
         @Config("allow_quoted_newlines")
         @ConfigDefault("false")
         boolean getAllowQuotedNewlines();
+
+        @Config("mode")
+        @ConfigDefault("\"append\"")
+        Mode getMode();
     }
 
     private final Logger log = Exec.getLogger(BigqueryOutputPlugin.class);
+    private final static String temporaryTableSuffix = Long.toString(System.currentTimeMillis());
     private static BigqueryWriter bigQueryWriter;
 
     @Override
@@ -182,6 +187,18 @@ public class BigqueryOutputPlugin
             }
         }
 
+        if (task.getMode().isReplaceMode()) {
+            if (task.getIsSkipJobResultCheck()) {
+                throw new ConfigException("If mode is replace or replace_backup, is_skip_job_result_check must be false");
+            }
+        }
+
+        if (task.getMode().isDeleteInAdvance()) {
+            if (!task.getAutoCreateTable()) {
+                throw new ConfigException("If mode is delete_in_advance, auto_create_table must be true");
+            }
+        }
+
         try {
             bigQueryWriter = new BigqueryWriter.Builder (
                     task.getAuthMethod().getString(),
@@ -217,8 +234,39 @@ public class BigqueryOutputPlugin
                              int taskCount,
                              FileOutputPlugin.Control control)
     {
+        Mode mode = taskSource.get(Mode.class, "Mode");
+        String project = taskSource.get(String.class, "Project");
+        String dataset = taskSource.get(String.class, "Dataset");
+        String tableName = taskSource.get(String.class, "Table");
+
+        if (mode == Mode.delete_in_advance) {
+            try {
+                bigQueryWriter.deleteTable(project, dataset, generateTableName(tableName));
+            } catch (IOException ex) {
+                log.warn(ex.getMessage());
+            }
+        }
+
         control.run(taskSource);
 
+        if (mode.isReplaceMode()) {
+            try {
+                if (mode == Mode.replace_backup && bigQueryWriter.isExistTable(project, dataset, generateTableName(tableName))) {
+                    bigQueryWriter.replaceTable(project, dataset, generateTableName(tableName) + "_old", generateTableName(tableName));
+                }
+                bigQueryWriter.replaceTable(project, dataset, generateTableName(tableName), generateTemporaryTableName(tableName));
+            } catch (TimeoutException | BigqueryWriter.JobFailedException | IOException ex) {
+                log.error(ex.getMessage());
+                throw Throwables.propagate(ex);
+            } finally {
+                try {
+                    bigQueryWriter.deleteTable(project, dataset, generateTemporaryTableName(tableName));
+                } catch (IOException ex) {
+                    log.warn(ex.getMessage());
+                }
+            }
+        }
+
         return Exec.newConfigDiff();
     }
 
@@ -252,7 +300,8 @@ public class BigqueryOutputPlugin
         return new TransactionalFileOutput() {
             private final String project = task.getProject();
             private final String dataset = task.getDataset();
-            private final String table = generateTableName(task.getTable());
+            private final String table = task.getMode().isReplaceMode() ?
+                    generateTemporaryTableName(task.getTable()) : generateTableName(task.getTable());
             private final boolean deleteFromLocalWhenJobEnd = task.getDeleteFromLocalWhenJobEnd();
 
             private int fileIndex = 0;
@@ -351,6 +400,11 @@ public class BigqueryOutputPlugin
         return result.toString();
     }
 
+    public String generateTemporaryTableName(String tableName)
+    {
+        return generateTableName(tableName) + temporaryTableSuffix;
+    }
+
     public enum SourceFormat
     {
         CSV("CSV"),
@@ -387,4 +441,32 @@ public class BigqueryOutputPlugin
             return string;
         }
     }
+
+    public enum Mode
+    {
+        append("append"),
+        delete_in_advance("delete_in_advance") {
+            @Override
+            public boolean isDeleteInAdvance() { return true; }
+        },
+        replace("replace") {
+            @Override
+            public boolean isReplaceMode() { return true; }
+        },
+        replace_backup("replace_backup") {
+            @Override
+            public boolean isReplaceMode() { return true; }
+        };
+
+        private final String string;
+
+        Mode(String string)
+        {
+            this.string = string;
+        }
+
+        public String getString() { return string; }
+        public boolean isReplaceMode() { return false; }
+        public boolean isDeleteInAdvance() { return true; }
+    }
 }

data/src/main/java/org/embulk/output/BigqueryWriter.java

@@ -11,6 +11,8 @@ import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.List;
 import java.util.concurrent.TimeoutException;
+
+import com.google.api.services.bigquery.model.*;
 import com.google.common.base.Optional;
 import java.security.GeneralSecurityException;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -24,16 +26,6 @@ import org.slf4j.Logger;
 import com.google.api.services.bigquery.Bigquery;
 import com.google.api.services.bigquery.Bigquery.Tables;
 import com.google.api.services.bigquery.Bigquery.Jobs.Insert;
-import com.google.api.services.bigquery.model.Job;
-import com.google.api.services.bigquery.model.JobConfiguration;
-import com.google.api.services.bigquery.model.JobConfigurationLoad;
-import com.google.api.services.bigquery.model.JobStatistics;
-import com.google.api.services.bigquery.model.JobReference;
-import com.google.api.services.bigquery.model.Table;
-import com.google.api.services.bigquery.model.TableSchema;
-import com.google.api.services.bigquery.model.TableReference;
-import com.google.api.services.bigquery.model.TableFieldSchema;
-import com.google.api.services.bigquery.model.ErrorProto;
 import com.google.api.client.googleapis.json.GoogleJsonResponseException;
 import com.google.api.client.googleapis.media.MediaHttpUploader;
 import com.google.api.client.googleapis.media.MediaHttpUploaderProgressListener;
@@ -190,6 +182,44 @@ public class BigqueryWriter
         }
     }
 
+    public void replaceTable(String project, String dataset, String oldTable, String newTable)
+            throws TimeoutException, JobFailedException, IOException
+    {
+        copyTable(project, dataset, newTable, oldTable, false);
+    }
+
+    public void copyTable(String project, String dataset, String fromTable, String toTable, boolean append)
+            throws TimeoutException, JobFailedException, IOException
+    {
+        log.info(String.format("Copy Job preparing... project:%s dataset:%s from:%s to:%s", project, dataset, fromTable, toTable));
+
+        Job job = new Job();
+        JobReference jobRef = null;
+        JobConfiguration jobConfig = new JobConfiguration().setCopy(setCopyConfig(project, dataset, fromTable, toTable, append));
+        job.setConfiguration(jobConfig);
+        Insert insert = bigQueryClient.jobs().insert(project, job);
+        insert.setProjectId(project);
+        insert.setDisableGZipContent(true);
+
+        try {
+            jobRef = insert.execute().getJobReference();
+        } catch (IllegalStateException ex) {
+            throw new JobFailedException(ex.getMessage());
+        }
+        log.info(String.format("Job executed. job id:[%s]", jobRef.getJobId()));
+        getJobStatusUntilDone(project, jobRef);
+    }
+
+    public void deleteTable(String project, String dataset, String table) throws IOException {
+        try {
+            Tables.Delete delete = bigQueryClient.tables().delete(project, dataset, table);
+            delete.execute();
+            log.info(String.format("Table deleted. project:%s dataset:%s table:%s", delete.getProjectId(), delete.getDatasetId(), delete.getTableId()));
+        } catch (GoogleJsonResponseException ex) {
+            log.warn(ex.getMessage());
+        }
+    }
+
     private JobConfigurationLoad setLoadConfig(String project, String dataset, String table)
     {
         JobConfigurationLoad config = new JobConfigurationLoad();
@@ -214,6 +244,21 @@ public class BigqueryWriter
         return config;
     }
 
+    private JobConfigurationTableCopy setCopyConfig(String project, String dataset, String fromTable, String toTable, boolean append)
+    {
+        JobConfigurationTableCopy config = new JobConfigurationTableCopy();
+        config.setSourceTable(createTableReference(project, dataset, fromTable))
+                .setDestinationTable(createTableReference(project, dataset, toTable));
+
+        if (append) {
+            config.setWriteDisposition("WRITE_APPEND");
+        } else {
+            config.setWriteDisposition("WRITE_TRUNCATE");
+        }
+
+        return config;
+    }
+
     private String createJobId(ImmutableList<String> elements) throws NoSuchAlgorithmException, IOException
     {
         StringBuilder sb = new StringBuilder();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.2.0
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-16 00:00:00.000000000 Z
+date: 2016-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -63,7 +63,7 @@ files:
 - src/test/java/org/embulk/output/TestBigqueryWriter.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-output-bigquery-0.1.11.jar
+- classpath/embulk-output-bigquery-0.2.0.jar
 - classpath/google-api-client-1.20.0.jar
 - classpath/google-api-services-bigquery-v2-rev205-1.20.0.jar
 - classpath/google-http-client-1.20.0.jar