embulk-output-bigquery 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b37d638ca9c217221687cdcadfbd45257291aef4
-  data.tar.gz: 972bf78e9ce75972fd3f2e1f77389a7383c3d2a0
+  metadata.gz: 37643ccea137f84b59a056d75826b54c8548c0b6
+  data.tar.gz: 1eafbf6ab69c81039c22712cda93ceb099ec9d92
 SHA512:
-  metadata.gz: 6d18639e76da80f45e2852df8408ec4c9c655e06a77a776c42dfb52310cc787e8319fd8f182eef20c59ffd01e5939ccb3799b4a7bad5b20f416aa668d513b3e3
-  data.tar.gz: 5a37cded1558ba6f3fbb1d4c475c46a593151a32e831299c00c386185d32828ec72810c2334863f7e03cdc7a3b2d974e7d1ec0cf653276073e7d336398ae92ba
+  metadata.gz: 3f8e1a7b70fb1a71060338b5aa2c4676a02ee56289fe5f0470fda09081ae2cec3b5c4fece89993beab298b43fb3769b3db8d28b485a4abacca40b57fdb6f3759
+  data.tar.gz: d4b1aab992230a263642b6eb0cac08d1981d6cf54728ef76fec84da323888172ba6c6ed891f1cb7914f95dc9887ee21856d43e0fb044fa3c8e1aafd7cc8ab428

data/README.md

@@ -36,6 +36,7 @@ OAuth flow for installed applications.
 - **table**: table name (string, required)
 - **auto_create_table**: (boolean, optional default is 0)
 - **schema_path**: (string, optional)
+- **prevent_duplicate_insert**: (boolean, optional default is 0)
 - **application_name**: application name anything you like (string, optional)
 - **delete_from_local_when_job_end**: (boolean, optional, default is 0)
 - **job_status_max_polling_time**: max job status polling time. (int, optional, default is 3600 sec)
@@ -120,6 +121,22 @@ out:
   schema_path: /path/to/schema.json
 ```
 
+### Data Consistency
+
+When `prevent_duplicate_insert` is set to true, embulk-output-bigquery generate job ID from md5 hash of file  and other options to prevent duplicate data insertion.
+
+`job ID = md5(md5(file) + dataset + table + schema + source_format + file_delimiter + max_bad_records + encoding)`
+
+[job ID must be unique(including failures)](https://cloud.google.com/bigquery/loading-data-into-bigquery#consistency). So same data can't insert with same settings.
+
+In other words, you can retry as many times as you like, in case something bad error(like network error) happens before job insertion.
+
+```yaml
+out:
+  type: bigquery
+  prevent_duplicate_insert: true
+```
+
 ## Build
 
 ```

data/build.gradle

@@ -15,14 +15,14 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.3"
+version = "0.1.4"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.5.1"
     provided "org.embulk:embulk-core:0.5.1"
 
-    compile "com.google.http-client:google-http-client-jackson2:1.19.0"
-    compile "com.google.apis:google-api-services-bigquery:v2-rev193-1.19.1"
+    compile "com.google.http-client:google-http-client-jackson2:1.20.0"
+    compile "com.google.apis:google-api-services-bigquery:v2-rev205-1.20.0"
 
     testCompile "junit:junit:4.+"
 }

data/src/main/java/org/embulk/output/BigqueryOutputPlugin.java

@@ -5,6 +5,7 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.BufferedOutputStream;
 import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
 import java.util.List;
 import java.util.concurrent.TimeoutException;
 import com.google.common.base.Optional;
@@ -96,6 +97,10 @@ public class BigqueryOutputPlugin
         @ConfigDefault("null")
         public Optional<String> getSchemaPath();
 
+        @Config("prevent_duplicate_insert")
+        @ConfigDefault("false")
+        public boolean getPreventDuplicateInsert();
+
         @Config("job_status_max_polling_time")
         @ConfigDefault("3600")
         public int getJobStatusMaxPollingTime();
@@ -105,7 +110,7 @@ public class BigqueryOutputPlugin
         public int getJobStatusPollingInterval();
 
         @Config("is_skip_job_result_check")
-        @ConfigDefault("0")
+        @ConfigDefault("false")
         public boolean getIsSkipJobResultCheck();
     }
 
@@ -131,6 +136,7 @@ public class BigqueryOutputPlugin
                     .setFieldDelimiter(task.getFieldDelimiter())
                     .setMaxBadrecords(task.getMaxBadrecords())
                     .setEncoding(task.getEncoding())
+                    .setPreventDuplicateInsert(task.getPreventDuplicateInsert())
                     .setJobStatusMaxPollingTime(task.getJobStatusMaxPollingTime())
                     .setJobStatusPollingInterval(task.getJobStatusPollingInterval())
                     .setIsSkipJobResultCheck(task.getIsSkipJobResultCheck())
@@ -233,7 +239,8 @@ public class BigqueryOutputPlugin
                             log.info(String.format("Delete local file [%s]", filePath));
                             file.delete();
                         }
-                    } catch (IOException | TimeoutException | BigqueryWriter.JobFailedException ex) {
+                    } catch (NoSuchAlgorithmException | TimeoutException | BigqueryWriter.JobFailedException | IOException ex) {
+                        log.error(ex.getMessage());
                         throw Throwables.propagate(ex);
                     }
                 }

data/src/main/java/org/embulk/output/BigqueryWriter.java

@@ -6,14 +6,18 @@ import java.io.FileNotFoundException;
 import java.io.FileInputStream;
 import java.io.BufferedInputStream;
 import com.google.api.client.http.InputStreamContent;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.List;
 import java.util.concurrent.TimeoutException;
 import com.google.common.base.Optional;
+import com.google.api.client.util.Base64;
 import com.google.common.base.Throwables;
 import java.security.GeneralSecurityException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.core.type.TypeReference;
 
+import org.apache.commons.codec.binary.Hex;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
 
@@ -48,6 +52,7 @@ public class BigqueryWriter
     private final String fieldDelimiter;
     private final int maxBadrecords;
     private final String encoding;
+    private final boolean preventDuplicateInsert;
     private final long jobStatusMaxPollingTime;
     private final long jobStatusPollingInterval;
     private final boolean isSkipJobResultCheck;
@@ -64,6 +69,7 @@ public class BigqueryWriter
         this.fieldDelimiter = builder.fieldDelimiter;
         this.maxBadrecords = builder.maxBadrecords;
         this.encoding = builder.encoding.toUpperCase();
+        this.preventDuplicateInsert = builder.preventDuplicateInsert;
         this.jobStatusMaxPollingTime = builder.jobStatusMaxPollingTime;
         this.jobStatusPollingInterval = builder.jobStatusPollingInterval;
         this.isSkipJobResultCheck = builder.isSkipJobResultCheck;
@@ -91,7 +97,7 @@ public class BigqueryWriter
             List<ErrorProto> errors = job.getStatus().getErrors();
             if (errors != null) {
                 for (ErrorProto error : errors) {
-                    log.warn(String.format("Error: job id:[%s] reason[%s][%s] location:[%s]", jobRef.getJobId(), error.getReason(), error.getMessage(), error.getLocation()));
+                    log.error(String.format("Error: job id:[%s] reason[%s][%s] location:[%s]", jobRef.getJobId(), error.getReason(), error.getMessage(), error.getLocation()));
                 }
             }
 
@@ -132,17 +138,24 @@ public class BigqueryWriter
         }
     }
 
-    public void executeLoad(String localFilePath) throws GoogleJsonResponseException, IOException, TimeoutException, JobFailedException
+    public void executeLoad(String localFilePath) throws GoogleJsonResponseException, NoSuchAlgorithmException,
+            TimeoutException, JobFailedException, IOException
     {
         log.info(String.format("Job preparing... project:%s dataset:%s table:%s", project, dataset, table));
 
         Job job = new Job();
-        JobReference jobRef = null;
+        JobReference jobRef = new JobReference();
         JobConfiguration jobConfig = new JobConfiguration();
         JobConfigurationLoad loadConfig = new JobConfigurationLoad();
         jobConfig.setLoad(loadConfig);
         job.setConfiguration(jobConfig);
 
+        if (preventDuplicateInsert) {
+            String jobId = createJobId(localFilePath);
+            jobRef.setJobId(jobId);
+            job.setJobReference(jobRef);
+        }
+
         loadConfig.setAllowQuotedNewlines(false);
         loadConfig.setEncoding(encoding);
         loadConfig.setMaxBadRecords(maxBadrecords);
@@ -181,9 +194,8 @@ public class BigqueryWriter
 
         try {
             jobRef = insert.execute().getJobReference();
-        } catch (Exception ex) {
-            log.warn("Job execution was failed. Please check your settings or data... like data matches schema");
-            throw Throwables.propagate(ex);
+        } catch (IllegalStateException ex) {
+            throw new JobFailedException(ex.getMessage());
         }
         log.info(String.format("Job executed. job id:[%s] file:[%s]", jobRef.getJobId(), localFilePath));
         if (isSkipJobResultCheck) {
@@ -193,6 +205,25 @@ public class BigqueryWriter
         }
     }
 
+    private String createJobId(String localFilePath) throws NoSuchAlgorithmException, IOException
+    {
+        StringBuilder sb = new StringBuilder();
+        sb.append(getLocalMd5hash(localFilePath));
+        sb.append(dataset);
+        sb.append(table);
+        sb.append(tableSchema);
+        sb.append(sourceFormat);
+        sb.append(fieldDelimiter);
+        sb.append(maxBadrecords);
+        sb.append(encoding);
+
+        MessageDigest md = MessageDigest.getInstance("MD5");
+        String str = new String(sb);
+        byte[] digest = md.digest(str.getBytes());
+        String hash = new String(Hex.encodeHex(digest));
+        return "embulk_job_" + hash;
+    }
+
     private TableReference createTableReference()
     {
         return new TableReference()
@@ -248,6 +279,28 @@ public class BigqueryWriter
         }
     }
 
+    private String getLocalMd5hash(String filePath) throws NoSuchAlgorithmException, IOException
+    {
+        FileInputStream stream = null;
+        try {
+            stream = new FileInputStream(filePath);
+            MessageDigest digest = MessageDigest.getInstance("MD5");
+
+            byte[] bytesBuffer = new byte[1024];
+            int bytesRead = -1;
+
+            while ((bytesRead = stream.read(bytesBuffer)) != -1) {
+                digest.update(bytesBuffer, 0, bytesRead);
+            }
+            byte[] hashedBytes = digest.digest();
+
+            byte[] encoded = (hashedBytes);
+            return new String(encoded);
+        } finally {
+            stream.close();
+        }
+    }
+
     private class UploadProgressListener implements MediaHttpUploaderProgressListener
     {
         private String fileName;
@@ -291,6 +344,7 @@ public class BigqueryWriter
         private String fieldDelimiter;
         private int maxBadrecords;
         private String encoding;
+        private boolean preventDuplicateInsert;
         private int jobStatusMaxPollingTime;
         private int jobStatusPollingInterval;
         private boolean isSkipJobResultCheck;
@@ -372,6 +426,12 @@ public class BigqueryWriter
             return this;
         }
 
+        public Builder setPreventDuplicateInsert(boolean preventDuplicateInsert)
+        {
+            this.preventDuplicateInsert = preventDuplicateInsert;
+            return this;
+        }
+
         public Builder setJobStatusMaxPollingTime(int jobStatusMaxPollingTime)
         {
             this.jobStatusMaxPollingTime = jobStatusMaxPollingTime;
@@ -396,7 +456,7 @@ public class BigqueryWriter
         }
     }
 
-    public class JobFailedException extends Exception
+    public class JobFailedException extends RuntimeException
     {
         public JobFailedException(String message) {
             super(message);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-06 00:00:00.000000000 Z
+date: 2015-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,12 +62,12 @@ files:
 - src/test/java/org/embulk/output/TestBigqueryWriter.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-output-bigquery-0.1.3.jar
-- classpath/google-api-client-1.19.1.jar
-- classpath/google-api-services-bigquery-v2-rev193-1.19.1.jar
-- classpath/google-http-client-1.19.0.jar
-- classpath/google-http-client-jackson2-1.19.0.jar
-- classpath/google-oauth-client-1.19.0.jar
+- classpath/embulk-output-bigquery-0.1.4.jar
+- classpath/google-api-client-1.20.0.jar
+- classpath/google-api-services-bigquery-v2-rev205-1.20.0.jar
+- classpath/google-http-client-1.20.0.jar
+- classpath/google-http-client-jackson2-1.20.0.jar
+- classpath/google-oauth-client-1.20.0.jar
 - classpath/guava-jdk5-13.0.jar
 - classpath/httpclient-4.0.1.jar
 - classpath/httpcore-4.0.1.jar