embulk-input-bigquery_extract_files 0.0.12 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: debc8d2786db90b41c8e777dc32665299aa3200b
-  data.tar.gz: e14e906f62b553b73f1beaad28a615ae96f9ce76
+  metadata.gz: bdb3cb2064c1b19c0748adb981d2593fd30950af
+  data.tar.gz: 4c54a7075d3b4877e0ad66d2e91125fbd2b1af61
 SHA512:
-  metadata.gz: dfa41fc406217af65e6d99009013d1c6d0f367ea1f8639fef7160449f6e54a1312b5ce6ef5fbc76dabf7735e5ea0b5937445cb51255940a1184338f9bc9a9265
-  data.tar.gz: 40b6ea3bb6bfad97302efac14e435698113bc6c463221aac50d6c3a991203ae53e0b0cbd731c9a9d21bb6916179f504c0dc108fa3cc4519eaa3226806a5cd524
+  metadata.gz: 8e4c7646e18867ef61a6488151eed43bda4001b1935bcffc1d5cb5ef545eaf731f3aa94f3f663a07d8d9e363f60b407072dcfea4c41722c5cd8c4dcb807e48cf
+  data.tar.gz: c5f6d6843afb3848714be3bb0ae9dc249ea68f40cf92db75911222c7d72adf0bd55f4b65c8cb1971a8eb357e8678425d0a06befc6d9fb7e0a6668c93da0c04a3

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.12"
+version = "0.0.13"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java

@@ -122,7 +122,8 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
         @Config("bigquery_job_wait_second")
         @ConfigDefault("600")
         public Optional<Integer> getBigqueryJobWaitingSecond();
-        
+        public void setBigqueryJobWaitingSecond(Optional<Integer> second);
+
         @Config("cleanup_gcs_files")
         @ConfigDefault("false")
         public boolean getCleanupGcsTempFiles(); 
@@ -174,6 +175,16 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
         public boolean getDirectDownloadEnabled();
         //public Schema getSchemaConfig();
         //public void setSchameConfig(SchemaConfig schema);
+
+        /**
+         * 2020.11.16 sometimes, bigquery extract job just very slow. not becouse of error. then workflow must throw exception for alert message
+         * @return
+         */
+        @Config("throw_bigquery_job_wait_timeout")
+        @ConfigDefault("false")
+        public boolean getThrowBigqueryJobWaitTimeout();
+        public void setThrowBigqueryJobWaitTimeout(boolean toThrow);
+
     }
 
 	@Override

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java

@@ -1,11 +1,6 @@
 package org.embulk.input.bigquery_export_gcs;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
 import java.math.BigInteger;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
@@ -141,7 +136,7 @@ public class BigqueryExportUtils
 
 		log.info("query to Table jobId : {} : waiting for job end...",jobId);
 		
-		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get());
+		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get(), task.getThrowBigqueryJobWaitTimeout());
 		
 		log.debug("waiting for job end....... {}", lastJob.toPrettyString());
 	}
@@ -220,7 +215,10 @@ public class BigqueryExportUtils
         do {
           objects = listRequest.execute();
           if(objects.getItems() == null){
-        	  log.error("file not found in gs://{}/{}",bucket,blobName);
+
+          	  String errorMessage = String.format("file not found in gs://%s/%s",bucket,blobName);
+			  log.error(errorMessage);
+
         	  return builder.build();
           }
           for(StorageObject obj : objects.getItems()){
@@ -337,14 +335,14 @@ public class BigqueryExportUtils
 		log.info("extract jobId : {}",jobId);
 		log.debug("waiting for job end....... ");
 		
-		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get());
+		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get(), task.getThrowBigqueryJobWaitTimeout());
 		
 		log.info("table extract result : {}",lastJob.toPrettyString());
 		
 		return embulkSchema;
     }
 
-    public static Job waitForJob(Bigquery bigquery, String project, String jobId, String location, int bigqueryJobWaitingSecond) throws IOException, InterruptedException{
+    public static Job waitForJob(Bigquery bigquery, String project, String jobId, String location, int bigqueryJobWaitingSecond, boolean exceptionWhenTimeout) throws IOException, InterruptedException{
     	int maxAttempts = bigqueryJobWaitingSecond;
 		int initialRetryDelay = 1000; // ms
 		Job pollingJob = null;	
@@ -358,11 +356,18 @@ public class BigqueryExportUtils
             if (pollingJob.getStatus().getState().equals("DONE")) {
                 break;
             }
-            log.info("waiting {} ... ",tryCnt);
+            log.info("waiting {} ... {} ", tryCnt,state);
             Thread.sleep(initialRetryDelay);
         }
-        if(tryCnt + 1 == maxAttempts){
-        	log.error("Bigquery Job Waiting exceed : over {} second...", bigqueryJobWaitingSecond);
+        if(tryCnt + 1 >= maxAttempts){
+
+			String errorMessage = String.format("Bigquery Job [%s] Waiting timeout : over %s second...", jobId, bigqueryJobWaitingSecond);
+			if(exceptionWhenTimeout){
+				throw new IOException(errorMessage);
+			}else{
+				log.error(errorMessage);
+			}
+
         }
         
         return pollingJob;

data/src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java

@@ -4,10 +4,13 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 
+import com.google.common.base.Optional;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.validation.constraints.AssertTrue;
+
 public class TestGoogleCloudAccessData  extends UnitTestInitializer
 {
 	private static final Logger log = LoggerFactory.getLogger(TestGoogleCloudAccessData.class);
@@ -23,6 +26,19 @@ public class TestGoogleCloudAccessData  extends UnitTestInitializer
     	
     	log.info("file size : {}",org.apache.commons.compress.utils.IOUtils.toByteArray(ins).length);
     }
-    
+
+
+    @Test(expected=Exception.class)
+    public void testJobWaitTimeout() throws FileNotFoundException, IOException
+    {
+        BigqueryExportGcsFileInputPlugin.PluginTask task = config.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
+        task.setThrowBigqueryJobWaitTimeout(true);
+        task.setBigqueryJobWaitingSecond(Optional.of(1));
+        plugin.executeBigqueryApi(task);
+
+        InputStream ins = BigqueryExportUtils.openInputStream(task, task.getFiles().get(0));
+        log.info("file size : {}",org.apache.commons.compress.utils.IOUtils.toByteArray(ins).length);
+
+    }
 
 }

data/src/test/java/org/embulk/input/bigquery_export_gcs/TestPluginFunctions.java

@@ -53,4 +53,5 @@ public class TestPluginFunctions extends UnitTestInitializer
     	assertEquals("", "bbb/ccc_", task.getGcsBlobNamePrefix());
     }
 
+
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-bigquery_extract_files
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
 platform: ruby
 authors:
 - jo8937
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-28 00:00:00.000000000 Z
+date: 2020-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,7 +65,7 @@ files:
 - src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java
 - classpath/animal-sniffer-annotations-1.14.jar
 - classpath/checker-compat-qual-2.5.2.jar
-- classpath/embulk-input-bigquery_extract_files-0.0.12.jar
+- classpath/embulk-input-bigquery_extract_files-0.0.13.jar
 - classpath/error_prone_annotations-2.1.3.jar
 - classpath/google-api-client-1.25.0.jar
 - classpath/google-api-services-bigquery-v2-rev429-1.25.0.jar