embulk-input-bigquery_extract_files 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3ad9b4688bba442662baac2209f35077b14c669b
-  data.tar.gz: 4b184864e80181b615cfef548ea065e6c2140a0d
+  metadata.gz: 23ea87433e2ed5453de192fc5ea73c4e1b7499b6
+  data.tar.gz: f28b4f9530974d7f2614392f951823af1213ed4a
 SHA512:
-  metadata.gz: f6617b86ba4e1dff3cf2fb689c8b770b12a3c71598e5b2d68269037b456f18bb29a576f127bacdf3d088913912c6eb2f6f1305a19d17f66543ef5a769668f7bf
-  data.tar.gz: c5394ab788b193dfc1920b508a715453ae30c40c1a629b3582ea8066778ee2ff965872b1a5b3f1cd2230892be958bb8a22fe79551481f958591ebb09abee90f1
+  metadata.gz: 54384d90f6fab9406c1d56aa4f8eff7fb3ed78aff8564bf910f8aa36610447bfc878e4f025a8a466e4e14afdca0457a5ca087e8cb62151744b786635179bfd8c
+  data.tar.gz: 775aff297708f182088b36ad48664d70ae0a4a411184ba6ae47cf719651c32b9ef582f3037b1d75d6dac7288511cc9420179d02d052553a50748f404bcbebda3

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.3"
+version = "0.0.4"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java

@@ -116,10 +116,13 @@ public class BigqueryExportGcsFileInputPlugin
         @ConfigDefault("null")
         public Optional<String> getTempSchemaFilePath();
         
-        @Config("temp_schema_file_tpye")
+        @Config("temp_schema_file_type")
         @ConfigDefault("null")
         public Optional<String> getTempSchemaFileType();
         
+        @Config("bigquery_job_wait_second")
+        @ConfigDefault("600")
+        public Optional<Integer> getBigqueryJobWaitingSecond();
         
         @Config("cleanup_gcs_files")
         @ConfigDefault("false")

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java

@@ -133,7 +133,7 @@ public class BigqueryExportUtils
 		log.info("query to Table jobId : {}",jobId);
 		log.info("waiting for job end....... ");
 		
-		Job lastJob = waitForJob(bigquery, task.getProject(), jobId);
+		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getBigqueryJobWaitingSecond().get());
 		
 		log.debug("waiting for job end....... {}", lastJob.toPrettyString());
 	}
@@ -301,25 +301,31 @@ public class BigqueryExportUtils
 		log.info("extract jobId : {}",jobId);
 		log.debug("waiting for job end....... ");
 		
-		waitForJob(bigquery, task.getProject(), jobId);
+		waitForJob(bigquery, task.getProject(), jobId, task.getBigqueryJobWaitingSecond().get());
 		return embulkSchema;
     }
 
-    public static Job waitForJob(Bigquery bigquery, String project, String jobId) throws IOException, InterruptedException{
-    	int maxAttempts = 20;
+    public static Job waitForJob(Bigquery bigquery, String project, String jobId, int bigqueryJobWaitingSecond) throws IOException, InterruptedException{
+    	int maxAttempts = bigqueryJobWaitingSecond;
 		int initialRetryDelay = 1000; // ms
 		Job pollingJob = null;	
-        for (int i=0; i < maxAttempts; i++){
+		log.info("waiting for job end : {}",jobId);
+		int tryCnt = 0;
+        for (tryCnt=0; tryCnt < maxAttempts; tryCnt++){
             pollingJob = bigquery.jobs().get(project, jobId).execute();
             String state = pollingJob.getStatus().getState();
             log.debug("Job Status {} : {}",jobId, state);
+            
             if (pollingJob.getStatus().getState().equals("DONE")) {
                 break;
             }
-            log.debug("wait 1 second and waiting for end ...");
+            log.info("waiting {} ... ",tryCnt);
             Thread.sleep(initialRetryDelay);
-            
         }
+        if(tryCnt + 1 == maxAttempts){
+        	log.error("Bigquery Job Waiting exceed : over {} second...", bigqueryJobWaitingSecond);
+        }
+        
         return pollingJob;
     }
     

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-bigquery_extract_files
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - jo8937
@@ -64,7 +64,7 @@ files:
 - src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-bigquery_extract_files-0.0.3.jar
+- classpath/embulk-input-bigquery_extract_files-0.0.4.jar
 - classpath/google-api-client-1.23.0.jar
 - classpath/google-api-services-bigquery-v2-rev363-1.23.0.jar
 - classpath/google-api-services-storage-v1-rev59-1.21.0.jar