embulk-input-bigquery_extract_files 0.0.13 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bdb3cb2064c1b19c0748adb981d2593fd30950af
-  data.tar.gz: 4c54a7075d3b4877e0ad66d2e91125fbd2b1af61
+  metadata.gz: be7d0d070196d522edcb8bf289e5d5156acfef52
+  data.tar.gz: da58c00653e0a96db45475fc2c8f91012d620c2b
 SHA512:
-  metadata.gz: 8e4c7646e18867ef61a6488151eed43bda4001b1935bcffc1d5cb5ef545eaf731f3aa94f3f663a07d8d9e363f60b407072dcfea4c41722c5cd8c4dcb807e48cf
-  data.tar.gz: c5f6d6843afb3848714be3bb0ae9dc249ea68f40cf92db75911222c7d72adf0bd55f4b65c8cb1971a8eb357e8678425d0a06befc6d9fb7e0a6668c93da0c04a3
+  metadata.gz: 1076f9ac8e7fca9c6ec6e4558310700591c31249d9ea6bd76022ad2787a877f908338df9c64c77f23edb005cfb1f651a449f580affc09b49081acb8bb05b4053
+  data.tar.gz: 5510bc7e0b676b152dcd6ab31bc56ee54a2a5fc8009e93fb1b09c928fd08c3ef1242ce85bfa7a6fb1423cf64998414d1a295def64776a6b43f39f1dff8a2c36b

data/README.md

@@ -4,9 +4,9 @@ embulk file input plugin.
 
 - embulk : http://www.embulk.org/docs/
 
-- embulk plugins : http://www.embulk.org/plugins/
+- embulk plugins : https://plugins.embulk.org/
 
-Read files stored in Google Cloud Storage that extracted from Google Cloud Bigquery's table or query result.
+Reads files stored on Google Cloud Storage that extracted from bigquery table or query result
 
 ## Overview
 
@@ -16,9 +16,9 @@ Read files stored in Google Cloud Storage that extracted from Google Cloud Bigqu
 
 ### Detail
 
-Read files stored in Google Cloud Storage, that exported from Google Cloud Bigquery's table or query result.
+Reads files stored on Google Cloud Storage that extracted from bigquery table or query result
 
-Maybe solution for very big data in bigquery.
+Maybe solution for download very big data in bigquery.
 
 If you set  **table** config without **query** config, 
 then just extract table to Google Cloud Storage.
@@ -26,6 +26,7 @@ then just extract table to Google Cloud Storage.
 If you set **query** config,
 then query result save to temp table and then extracted that temp table to Google Cloud Storage uri.
 see : https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.extract
+      
    
 ## Usage
 
@@ -35,6 +36,12 @@ see : https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuratio
 embulk gem install embulk-input-bigquery_extract_files
 ```
 
+### Update plugin (latest version : 0.0.13)
+
+```bash
+embulk gem update embulk-input-bigquery_extract_files
+```
+
 * rubygem url : https://rubygems.org/profiles/jo8937
 
 
@@ -64,6 +71,8 @@ embulk gem install embulk-input-bigquery_extract_files
 
 - **bigquery_job_wait_second**: bigquery job waiting second. (Optional) (string, default: `600`)
 
+- **throw_bigquery_job_wait_timeout**: throw exception when bigquery job waiting second timeout. (Optional) (string, default: `false`)
+
 - **cleanup_gcs_before_executing**: delete all file in gcs temp path before process start (Optional) (string, default: `true`)
 
 - **cleanup_gcs_files**: delete all file in gcs temp path after process end (Optional) (string, default: `false`)
@@ -164,6 +173,20 @@ out:
 $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 ```
 
+
+## Plugin maintenance 
+
+for old version user
+
+### Remove plugin specific version 
+
+```bash
+embulk gem uninstall embulk-input-bigquery_extract_files --version 0.0.13
+```
+
+* rubygem url : https://rubygems.org/profiles/jo8937
+
+
 # Another choice
 
 This plugin useful for file-input type. but maybe so complicated to use.

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.13"
+version = "0.0.14"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java

@@ -70,7 +70,8 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
         @Config("query")
         @ConfigDefault("null")
         public Optional<String> getQuery();
-        
+        public void setQuery(Optional<String> tempDataset);
+
         @Config("file_format")
         @ConfigDefault("\"CSV\"")
         public Optional<String> getFileFormat();
@@ -185,6 +186,17 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
         public boolean getThrowBigqueryJobWaitTimeout();
         public void setThrowBigqueryJobWaitTimeout(boolean toThrow);
 
+        /**
+         * 2020.11.18 sometime, bigquery job return "DONE" but include errors.
+         * DONE does not mean job success.
+         * https://cloud.google.com/bigquery/docs/running-jobs#bigquery_create_job-java
+         *
+         * @return
+         */
+        @Config("throw_bigquery_job_includes_error")
+        @ConfigDefault("false")
+        public boolean getThrowBigqueryJobIncludesError();
+        public void setThrowBigqueryJobIncludesError(boolean toThrow);
     }
 
 	@Override

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java

@@ -4,6 +4,7 @@ import java.io.*;
 import java.math.BigInteger;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
+import java.util.Collections;
 import java.util.Date;
 import java.util.List;
 import java.util.UUID;
@@ -136,7 +137,7 @@ public class BigqueryExportUtils
 
 		log.info("query to Table jobId : {} : waiting for job end...",jobId);
 		
-		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get(), task.getThrowBigqueryJobWaitTimeout());
+		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get(), task.getThrowBigqueryJobWaitTimeout(), task.getThrowBigqueryJobIncludesError());
 		
 		log.debug("waiting for job end....... {}", lastJob.toPrettyString());
 	}
@@ -335,14 +336,14 @@ public class BigqueryExportUtils
 		log.info("extract jobId : {}",jobId);
 		log.debug("waiting for job end....... ");
 		
-		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get(), task.getThrowBigqueryJobWaitTimeout());
+		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get(), task.getThrowBigqueryJobWaitTimeout(), task.getThrowBigqueryJobIncludesError());
 		
 		log.info("table extract result : {}",lastJob.toPrettyString());
 		
 		return embulkSchema;
     }
 
-    public static Job waitForJob(Bigquery bigquery, String project, String jobId, String location, int bigqueryJobWaitingSecond, boolean exceptionWhenTimeout) throws IOException, InterruptedException{
+    public static Job waitForJob(Bigquery bigquery, String project, String jobId, String location, int bigqueryJobWaitingSecond, boolean exceptionWhenTimeout, boolean exceptionWhenErrorResult) throws IOException, InterruptedException{
     	int maxAttempts = bigqueryJobWaitingSecond;
 		int initialRetryDelay = 1000; // ms
 		Job pollingJob = null;	
@@ -352,9 +353,16 @@ public class BigqueryExportUtils
             pollingJob = bigquery.jobs().get(project, jobId).setLocation(location).execute();
             String state = pollingJob.getStatus().getState();
             log.debug("Job Status {} : {}",jobId, state);
-            
+
+			// 2020-11-18  DONE is not means "no error" then, we must handle it explictly
+			if(exceptionWhenErrorResult){
+				if(pollingJob.getStatus().getErrorResult() != null){
+					throw new IOException(pollingJob.getStatus().getErrorResult().getMessage());
+				}
+			}
+
             if (pollingJob.getStatus().getState().equals("DONE")) {
-                break;
+            	break;
             }
             log.info("waiting {} ... {} ", tryCnt,state);
             Thread.sleep(initialRetryDelay);

data/src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java

@@ -28,6 +28,21 @@ public class TestGoogleCloudAccessData  extends UnitTestInitializer
     }
 
 
+    @Test(expected=Exception.class)
+    public void testJobDoneButError() throws FileNotFoundException, IOException
+    {
+        BigqueryExportGcsFileInputPlugin.PluginTask task = config.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
+        task.setThrowBigqueryJobWaitTimeout(true);
+        task.setThrowBigqueryJobIncludesError(true);
+        task.setQuery(Optional.of("select a from b"));
+        plugin.executeBigqueryApi(task);
+
+        InputStream ins = BigqueryExportUtils.openInputStream(task, task.getFiles().get(0));
+        log.info("file size : {}",org.apache.commons.compress.utils.IOUtils.toByteArray(ins).length);
+
+    }
+
+
     @Test(expected=Exception.class)
     public void testJobWaitTimeout() throws FileNotFoundException, IOException
     {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-bigquery_extract_files
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.14
 platform: ruby
 authors:
 - jo8937
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-16 00:00:00.000000000 Z
+date: 2020-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,7 +65,7 @@ files:
 - src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java
 - classpath/animal-sniffer-annotations-1.14.jar
 - classpath/checker-compat-qual-2.5.2.jar
-- classpath/embulk-input-bigquery_extract_files-0.0.13.jar
+- classpath/embulk-input-bigquery_extract_files-0.0.14.jar
 - classpath/error_prone_annotations-2.1.3.jar
 - classpath/google-api-client-1.25.0.jar
 - classpath/google-api-services-bigquery-v2-rev429-1.25.0.jar