embulk-input-bigquery_extract_files 0.0.9 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55f1e7dc3d4dcd5ebe905f42c289c4034702718d
-  data.tar.gz: eda2e0485a70f8fa63e6f4c292a792bd2af7633a
+  metadata.gz: be7d0d070196d522edcb8bf289e5d5156acfef52
+  data.tar.gz: da58c00653e0a96db45475fc2c8f91012d620c2b
 SHA512:
-  metadata.gz: 30debdbcaed3653c29a66a327d5799c4af50fb1783e20244b8aa8ca539dfbcef3a093d159d5678ae684bee2900af44aa7d884a3dc950cd79ecf3eab92eec652e
-  data.tar.gz: c7f538f8dcb10b1799e6c38d4e0e9c5477da032a13ce3bede7ba03830b98c7f7961b1c87e5d141c7313e2f1781a141b19466a942ab826cfe67bd2c481586c913
+  metadata.gz: 1076f9ac8e7fca9c6ec6e4558310700591c31249d9ea6bd76022ad2787a877f908338df9c64c77f23edb005cfb1f651a449f580affc09b49081acb8bb05b4053
+  data.tar.gz: 5510bc7e0b676b152dcd6ab31bc56ee54a2a5fc8009e93fb1b09c928fd08c3ef1242ce85bfa7a6fb1423cf64998414d1a295def64776a6b43f39f1dff8a2c36b

data/README.md

@@ -4,9 +4,9 @@ embulk file input plugin.
 
 - embulk : http://www.embulk.org/docs/
 
-- embulk plugins : http://www.embulk.org/plugins/
+- embulk plugins : https://plugins.embulk.org/
 
-Read files stored in Google Cloud Storage that extracted from Google Cloud Bigquery's table or query result.
+Reads files stored on Google Cloud Storage that extracted from bigquery table or query result
 
 ## Overview
 
@@ -16,9 +16,9 @@ Read files stored in Google Cloud Storage that extracted from Google Cloud Bigqu
 
 ### Detail
 
-Read files stored in Google Cloud Storage, that exported from Google Cloud Bigquery's table or query result.
+Reads files stored on Google Cloud Storage that extracted from bigquery table or query result
 
-Maybe solution for very big data in bigquery.
+Maybe solution for download very big data in bigquery.
 
 If you set  **table** config without **query** config, 
 then just extract table to Google Cloud Storage.
@@ -26,6 +26,7 @@ then just extract table to Google Cloud Storage.
 If you set **query** config,
 then query result save to temp table and then extracted that temp table to Google Cloud Storage uri.
 see : https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.extract
+      
    
 ## Usage
 
@@ -35,6 +36,12 @@ see : https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuratio
 embulk gem install embulk-input-bigquery_extract_files
 ```
 
+### Update plugin (latest version : 0.0.13)
+
+```bash
+embulk gem update embulk-input-bigquery_extract_files
+```
+
 * rubygem url : https://rubygems.org/profiles/jo8937
 
 
@@ -42,6 +49,7 @@ embulk gem install embulk-input-bigquery_extract_files
 
 - **project**: Google Cloud Platform (gcp) project id (string, required)
 - **json_keyfile**: gcp service account's private key with json (string, required)
+- **location**: location of bigquery dataset and temp_dataset. see : https://cloud.google.com/bigquery/docs/locations (Optional) (string, default: `US`)
 - **gcs_uri**: bigquery result saved uri. bucket and path names parsed from this uri.  (string, required)
 - **temp_local_path**: extract files download directory in local machine (string, required)
 
@@ -63,6 +71,8 @@ embulk gem install embulk-input-bigquery_extract_files
 
 - **bigquery_job_wait_second**: bigquery job waiting second. (Optional) (string, default: `600`)
 
+- **throw_bigquery_job_wait_timeout**: throw exception when bigquery job waiting second timeout. (Optional) (string, default: `false`)
+
 - **cleanup_gcs_before_executing**: delete all file in gcs temp path before process start (Optional) (string, default: `true`)
 
 - **cleanup_gcs_files**: delete all file in gcs temp path after process end (Optional) (string, default: `false`)
@@ -71,6 +81,8 @@ embulk gem install embulk-input-bigquery_extract_files
 
 - **cleanup_local_temp_files**: delete all file in local temp dir (Optional) (string, default: `true`)
 
+- **direct_download_enabled**: gcs download option. see : https://developers.google.com/api-client-library/java/google-api-java-client/reference/1.19.1/com/google/api/client/googleapis/media/MediaHttpDownloader#setDirectDownloadEnabled(boolean)  (Optional) (string, default: `false`)
+
 - **decoders**: embulk java-file-input plugin's default attribute. see : http://www.embulk.org/docs/built-in.html#gzip-decoder-plugin
 - **parser**: embulk java-file-input plugin's default .attribute see : http://www.embulk.org/docs/built-in.html#csv-parser-plugin
 
@@ -161,6 +173,20 @@ out:
 $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 ```
 
+
+## Plugin maintenance 
+
+for old version user
+
+### Remove plugin specific version 
+
+```bash
+embulk gem uninstall embulk-input-bigquery_extract_files --version 0.0.13
+```
+
+* rubygem url : https://rubygems.org/profiles/jo8937
+
+
 # Another choice
 
 This plugin useful for file-input type. but maybe so complicated to use.

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.9"
+version = "0.0.14"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -22,11 +22,12 @@ dependencies {
     compile ("org.embulk:embulk-core:0.8.36")
     provided("org.embulk:embulk-core:0.8.36")
     
-    compile "com.google.http-client:google-http-client-jackson2:1.21.0"
+    compile "com.google.guava:guava:16+"
+    compile "com.google.http-client:google-http-client-jackson2:1.28.0"
     //compile ('com.google.apis:google-api-services-bigquery:v2-rev363-1.23.0') {exclude module: "guava-jdk5"}
-    compile ('com.google.apis:google-api-services-bigquery:v2-rev402-1.25.0') {exclude module: "guava-jdk5"}
+    compile ('com.google.apis:google-api-services-bigquery:v2-rev429-1.25.0') {exclude module: "guava-jdk5"}
     //compile ("com.google.apis:google-api-services-storage:v1-rev59-1.21.0") {exclude module: "guava-jdk5"}
-    compile ("com.google.apis:google-api-services-storage:v1-rev136-1.25.0") {exclude module: "guava-jdk5"}
+    compile ("com.google.apis:google-api-services-storage:v1-rev150-1.25.0") {exclude module: "guava-jdk5"}
     
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.36:tests"

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java

@@ -1,9 +1,6 @@
 package org.embulk.input.bigquery_export_gcs;
 
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
 import java.nio.file.Path;
 import java.util.List;
 
@@ -29,8 +26,6 @@ import org.slf4j.Logger;
 import com.google.api.services.bigquery.Bigquery;
 import com.google.common.base.Optional;
 
-import io.airlift.slice.RuntimeIOException;
-
 /**
  * 
  * 
@@ -60,6 +55,10 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
         @Config("json_keyfile")
         public String getJsonKeyfile();
         
+        @Config("location")
+        @ConfigDefault("\"US\"")
+        public Optional<String> getLocation();
+
         @Config("dataset")
         @ConfigDefault("null")
         public Optional<String> getDataset();
@@ -71,7 +70,8 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
         @Config("query")
         @ConfigDefault("null")
         public Optional<String> getQuery();
-        
+        public void setQuery(Optional<String> tempDataset);
+
         @Config("file_format")
         @ConfigDefault("\"CSV\"")
         public Optional<String> getFileFormat();
@@ -123,7 +123,8 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
         @Config("bigquery_job_wait_second")
         @ConfigDefault("600")
         public Optional<Integer> getBigqueryJobWaitingSecond();
-        
+        public void setBigqueryJobWaitingSecond(Optional<Integer> second);
+
         @Config("cleanup_gcs_files")
         @ConfigDefault("false")
         public boolean getCleanupGcsTempFiles(); 
@@ -168,9 +169,34 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
 
         public String getWorkId();
         public void setWorkId(String temp);
-        
+
+
+        @Config("direct_download_enabled")
+        @ConfigDefault("false")
+        public boolean getDirectDownloadEnabled();
         //public Schema getSchemaConfig();
         //public void setSchameConfig(SchemaConfig schema);
+
+        /**
+         * 2020.11.16 sometimes, bigquery extract job just very slow. not becouse of error. then workflow must throw exception for alert message
+         * @return
+         */
+        @Config("throw_bigquery_job_wait_timeout")
+        @ConfigDefault("false")
+        public boolean getThrowBigqueryJobWaitTimeout();
+        public void setThrowBigqueryJobWaitTimeout(boolean toThrow);
+
+        /**
+         * 2020.11.18 sometime, bigquery job return "DONE" but include errors.
+         * DONE does not mean job success.
+         * https://cloud.google.com/bigquery/docs/running-jobs#bigquery_create_job-java
+         *
+         * @return
+         */
+        @Config("throw_bigquery_job_includes_error")
+        @ConfigDefault("false")
+        public boolean getThrowBigqueryJobIncludesError();
+        public void setThrowBigqueryJobIncludesError(boolean toThrow);
     }
 
 	@Override
@@ -194,7 +220,7 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
         	log.info("create local downlaod path : {}", localPath);
         	boolean ok = localPath.mkdirs();
         	if(!ok){
-        		throw new RuntimeIOException(new IOException("local path create fail : " + localPath));
+        		throw new UncheckedIOException (new IOException("local path create fail : " + localPath));
         	}
         }
 	}
@@ -246,7 +272,7 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
     		BigqueryExportUtils.executeQueryToDestinationWorkTable(bigquery, task);
 		} catch (IOException e) {
 			log.error("bigquery io error",e);
-			throw new RuntimeIOException(e);
+			throw new UncheckedIOException(e);
 		} catch (InterruptedException e) {
 			log.error("bigquery job error",e);
 			throw new RuntimeException(e);
@@ -260,7 +286,7 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
     		return schema;
 		} catch (IOException e) {
 			log.error("bigquery io error",e);
-			throw new RuntimeIOException(e);
+			throw new UncheckedIOException (e);
 		} catch (InterruptedException e) {
 			log.error("bigquery job error",e);
 			throw new RuntimeException(e);
@@ -276,7 +302,7 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
 			return BigqueryExportUtils.getFileListFromGcs(task);
 		} catch (IOException e) {
 			log.error("GCS api call error");
-			throw new RuntimeIOException(e);
+			throw new UncheckedIOException (e);
 		}
 		
     }

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java

@@ -1,14 +1,10 @@
 package org.embulk.input.bigquery_export_gcs;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
 import java.math.BigInteger;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
+import java.util.Collections;
 import java.util.Date;
 import java.util.List;
 import java.util.UUID;
@@ -33,7 +29,7 @@ import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.client.repackaged.com.google.common.base.Strings;
+import com.google.common.base.Strings;
 import com.google.api.services.bigquery.Bigquery;
 import com.google.api.services.bigquery.Bigquery.Jobs.Insert;
 import com.google.api.services.bigquery.Bigquery.Tabledata;
@@ -59,8 +55,6 @@ import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 
-import io.airlift.slice.RuntimeIOException;
-
 /**
  * 
  * 
@@ -143,7 +137,7 @@ public class BigqueryExportUtils
 
 		log.info("query to Table jobId : {} : waiting for job end...",jobId);
 		
-		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getBigqueryJobWaitingSecond().get());
+		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get(), task.getThrowBigqueryJobWaitTimeout(), task.getThrowBigqueryJobIncludesError());
 		
 		log.debug("waiting for job end....... {}", lastJob.toPrettyString());
 	}
@@ -222,7 +216,10 @@ public class BigqueryExportUtils
         do {
           objects = listRequest.execute();
           if(objects.getItems() == null){
-        	  log.error("file not found in gs://{}/{}",bucket,blobName);
+
+          	  String errorMessage = String.format("file not found in gs://%s/%s",bucket,blobName);
+			  log.error(errorMessage);
+
         	  return builder.build();
           }
           for(StorageObject obj : objects.getItems()){
@@ -339,32 +336,46 @@ public class BigqueryExportUtils
 		log.info("extract jobId : {}",jobId);
 		log.debug("waiting for job end....... ");
 		
-		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getBigqueryJobWaitingSecond().get());
+		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get(), task.getThrowBigqueryJobWaitTimeout(), task.getThrowBigqueryJobIncludesError());
 		
 		log.info("table extract result : {}",lastJob.toPrettyString());
 		
 		return embulkSchema;
     }
 
-    public static Job waitForJob(Bigquery bigquery, String project, String jobId, int bigqueryJobWaitingSecond) throws IOException, InterruptedException{
+    public static Job waitForJob(Bigquery bigquery, String project, String jobId, String location, int bigqueryJobWaitingSecond, boolean exceptionWhenTimeout, boolean exceptionWhenErrorResult) throws IOException, InterruptedException{
     	int maxAttempts = bigqueryJobWaitingSecond;
 		int initialRetryDelay = 1000; // ms
 		Job pollingJob = null;	
 		log.info("waiting for job end : {}",jobId);
 		int tryCnt = 0;
         for (tryCnt=0; tryCnt < maxAttempts; tryCnt++){
-            pollingJob = bigquery.jobs().get(project, jobId).execute();
+            pollingJob = bigquery.jobs().get(project, jobId).setLocation(location).execute();
             String state = pollingJob.getStatus().getState();
             log.debug("Job Status {} : {}",jobId, state);
-            
+
+			// 2020-11-18  DONE is not means "no error" then, we must handle it explictly
+			if(exceptionWhenErrorResult){
+				if(pollingJob.getStatus().getErrorResult() != null){
+					throw new IOException(pollingJob.getStatus().getErrorResult().getMessage());
+				}
+			}
+
             if (pollingJob.getStatus().getState().equals("DONE")) {
-                break;
+            	break;
             }
-            log.info("waiting {} ... ",tryCnt);
+            log.info("waiting {} ... {} ", tryCnt,state);
             Thread.sleep(initialRetryDelay);
         }
-        if(tryCnt + 1 == maxAttempts){
-        	log.error("Bigquery Job Waiting exceed : over {} second...", bigqueryJobWaitingSecond);
+        if(tryCnt + 1 >= maxAttempts){
+
+			String errorMessage = String.format("Bigquery Job [%s] Waiting timeout : over %s second...", jobId, bigqueryJobWaitingSecond);
+			if(exceptionWhenTimeout){
+				throw new IOException(errorMessage);
+			}else{
+				log.error(errorMessage);
+			}
+
         }
         
         return pollingJob;
@@ -393,7 +404,7 @@ public class BigqueryExportUtils
         	log.info("Start download : gs://{}/{} ...to ... {} ",task.getGcsBucket(), file, task.getTempLocalPath());
         	
     	    Storage.Objects.Get getObject = gcs.objects().get(task.getGcsBucket(), file);
-    	    getObject.getMediaHttpDownloader().setDirectDownloadEnabled(true);
+    	    getObject.getMediaHttpDownloader().setDirectDownloadEnabled( task.getDirectDownloadEnabled() );
     	     
     	    // return getObject.executeMediaAsInputStream() // direct InputStream ?? I Think this is faster then temp file. but ...
     	    

data/src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java

@@ -4,19 +4,17 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 
+import com.google.common.base.Optional;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.validation.constraints.AssertTrue;
+
 public class TestGoogleCloudAccessData  extends UnitTestInitializer
 {
 	private static final Logger log = LoggerFactory.getLogger(TestGoogleCloudAccessData.class);
 
-    @Test
-    public void envTest(){
-    	log.info("{}",System.getenv("GCP_PROJECT"));
-    }
-
     @Test
     public void testGcsInputStreamOpen() throws FileNotFoundException, IOException
     {
@@ -28,6 +26,34 @@ public class TestGoogleCloudAccessData  extends UnitTestInitializer
     	
     	log.info("file size : {}",org.apache.commons.compress.utils.IOUtils.toByteArray(ins).length);
     }
-    
+
+
+    @Test(expected=Exception.class)
+    public void testJobDoneButError() throws FileNotFoundException, IOException
+    {
+        BigqueryExportGcsFileInputPlugin.PluginTask task = config.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
+        task.setThrowBigqueryJobWaitTimeout(true);
+        task.setThrowBigqueryJobIncludesError(true);
+        task.setQuery(Optional.of("select a from b"));
+        plugin.executeBigqueryApi(task);
+
+        InputStream ins = BigqueryExportUtils.openInputStream(task, task.getFiles().get(0));
+        log.info("file size : {}",org.apache.commons.compress.utils.IOUtils.toByteArray(ins).length);
+
+    }
+
+
+    @Test(expected=Exception.class)
+    public void testJobWaitTimeout() throws FileNotFoundException, IOException
+    {
+        BigqueryExportGcsFileInputPlugin.PluginTask task = config.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
+        task.setThrowBigqueryJobWaitTimeout(true);
+        task.setBigqueryJobWaitingSecond(Optional.of(1));
+        plugin.executeBigqueryApi(task);
+
+        InputStream ins = BigqueryExportUtils.openInputStream(task, task.getFiles().get(0));
+        log.info("file size : {}",org.apache.commons.compress.utils.IOUtils.toByteArray(ins).length);
+
+    }
 
 }

data/src/test/java/org/embulk/input/bigquery_export_gcs/TestPluginFunctions.java

@@ -53,4 +53,5 @@ public class TestPluginFunctions extends UnitTestInitializer
     	assertEquals("", "bbb/ccc_", task.getGcsBlobNamePrefix());
     }
 
+
 }

data/src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java

@@ -83,7 +83,8 @@ public class UnitTestInitializer
                       config.get(String.class, "temp_local_path")
                       );
     }
-    
+
+
     @Test
     public void testInitTask(){
     	BigqueryExportGcsFileInputPlugin.PluginTask task = config.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-bigquery_extract_files
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.14
 platform: ruby
 authors:
 - jo8937
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-28 00:00:00.000000000 Z
+date: 2020-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -63,21 +63,23 @@ files:
 - src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java
 - src/test/java/org/embulk/input/bigquery_export_gcs/TestPluginFunctions.java
 - src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java
-- classpath/commons-codec-1.10.jar
-- classpath/commons-logging-1.2.jar
-- classpath/embulk-input-bigquery_extract_files-0.0.9.jar
+- classpath/animal-sniffer-annotations-1.14.jar
+- classpath/checker-compat-qual-2.5.2.jar
+- classpath/embulk-input-bigquery_extract_files-0.0.14.jar
+- classpath/error_prone_annotations-2.1.3.jar
 - classpath/google-api-client-1.25.0.jar
-- classpath/google-api-services-bigquery-v2-rev402-1.25.0.jar
-- classpath/google-api-services-storage-v1-rev136-1.25.0.jar
-- classpath/google-http-client-1.25.0.jar
-- classpath/google-http-client-jackson2-1.25.0.jar
+- classpath/google-api-services-bigquery-v2-rev429-1.25.0.jar
+- classpath/google-api-services-storage-v1-rev150-1.25.0.jar
+- classpath/google-http-client-1.28.0.jar
+- classpath/google-http-client-jackson2-1.28.0.jar
 - classpath/google-oauth-client-1.25.0.jar
-- classpath/guava-20.0.jar
-- classpath/httpclient-4.5.5.jar
-- classpath/httpcore-4.4.9.jar
+- classpath/grpc-context-1.14.0.jar
+- classpath/guava-26.0-android.jar
 - classpath/j2objc-annotations-1.1.jar
 - classpath/jackson-core-2.9.6.jar
 - classpath/jsr305-3.0.2.jar
+- classpath/opencensus-api-0.18.0.jar
+- classpath/opencensus-contrib-http-util-0.18.0.jar
 homepage: https://github.com/jo8937/embulk-input-bigquery_extract_files
 licenses:
 - MIT