embulk-input-bigquery_extract_files 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23ea87433e2ed5453de192fc5ea73c4e1b7499b6
-  data.tar.gz: f28b4f9530974d7f2614392f951823af1213ed4a
+  metadata.gz: 85c5df6d2dabf2727e267cabd33a2fd163e22942
+  data.tar.gz: 7b24ecc581b1b5c789ad4f33ab73303bc7c6ca8d
 SHA512:
-  metadata.gz: 54384d90f6fab9406c1d56aa4f8eff7fb3ed78aff8564bf910f8aa36610447bfc878e4f025a8a466e4e14afdca0457a5ca087e8cb62151744b786635179bfd8c
-  data.tar.gz: 775aff297708f182088b36ad48664d70ae0a4a411184ba6ae47cf719651c32b9ef582f3037b1d75d6dac7288511cc9420179d02d052553a50748f404bcbebda3
+  metadata.gz: 7b868edc052a4eec8cdd57fc5ae5899d25bb7fa65ab1289785e6c8e6f05e7d280a79e43ea608e31d9c883fc72776ea91b1a7cc1b381a0f4d898c651ebc203fae
+  data.tar.gz: 61c5c9616ca9b96ab7587144b25a16b7bdc59e34ebf9b6f38def677e080c8b7f5df26b75b725009df18cd35a96434d224a8644bba88510af7d307d429f1e0a54

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.4"
+version = "0.0.5"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/config.yml

@@ -25,6 +25,7 @@ out:
   port: 10315
   database: dbname
   table: test_table
-  options: {connectTimeout: 200000}
+  # https://dev.mysql.com/doc/connector-j/5.1/en/connector-j-reference-configuration-properties.html
+  options: {connectTimeout: 0, enableQueryTimeouts: false, waitTimeout: 0}
   mode: insert_direct
   

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java

@@ -1,6 +1,7 @@
 package org.embulk.input.bigquery_export_gcs;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Path;
@@ -132,6 +133,15 @@ public class BigqueryExportGcsFileInputPlugin
         @ConfigDefault("true")
         public boolean getCleanupTempTable(); 
         
+        @Config("cleanup_gcs_before_executing")
+        @ConfigDefault("true")
+        public boolean getCleanupGcsBeforeExcuting();
+        
+        
+        @Config("start_phase")
+        @ConfigDefault("0")
+        public int getStartPhase();
+        
         public List<String> getFiles();
         public void setFiles(List<String> files);
 
@@ -156,7 +166,7 @@ public class BigqueryExportGcsFileInputPlugin
         //public Schema getSchemaConfig();
         //public void setSchameConfig(SchemaConfig schema);
     }
-    
+        
     @Override
 	public ConfigDiff guess(ConfigSource execConfig, ConfigSource inputConfig) {
 
@@ -192,20 +202,39 @@ public class BigqueryExportGcsFileInputPlugin
 	
 	public void executeBigqueryApi(PluginTask task) {
 
+		log.info("[0] Initialize Settings ... ");
+		
         BigqueryExportUtils.parseGcsUri(task);
 
-        Schema schema = extractBigqueryToGcs(task);
+        if(task.getCleanupGcsBeforeExcuting()){
+			log.info("clean up before executing. delete all file in : {}",task.getGcsUri());
+			BigqueryExportUtils.removeGcsFilesBeforeExecuting(task);	
+		}
+
+        PHASE phase = BigqueryExportUtils.initTask(task);
+        log.info("Configuration : {}",task.toString());
         
+        Bigquery bigquery = BigqueryExportUtils.newBigqueryClient(task);
+       
+        if(phase == PHASE.QUERY){
+        	log.info("[1] Query to Table");
+        	extractQueryToTable(bigquery, task);
+        	
+        }
+        log.info("[2] Table to GCS");
+        Schema schema = extractTableToGcs(bigquery, task);
         log.info("Schema : {}",schema.toString());
         
+        log.info("[3] Write Schema ");
         writeSchemaFileIfSpecified(schema, task);
         
+        log.info("[4] read file list in gcs ");
         List<String> files = listFilesOfGcs(task);
         
         task.setFiles(files);
         
 	}
-	
+		
 	public void writeSchemaFileIfSpecified(Schema schema, PluginTask task) {
 		if(task.getTempSchemaFilePath().isPresent()) {
 			log.info("generate temp {} schema file to ... {}", task.getTempSchemaFileType().or(""), task.getTempSchemaFilePath().orNull());
@@ -213,16 +242,22 @@ public class BigqueryExportGcsFileInputPlugin
         }
 	}
 	
-    public Schema extractBigqueryToGcs(PluginTask task){
-    	try {
-    		Bigquery bigquery = BigqueryExportUtils.newBigqueryClient(task);
-    		
-    		// query init or execute query
-    		BigqueryExportUtils.initWorkTableWithExecuteQuery(bigquery,task);
-    		
+	public void extractQueryToTable(Bigquery bigquery, PluginTask task){
+		try {
+    		BigqueryExportUtils.executeQueryToDestinationWorkTable(bigquery, task);
+		} catch (IOException e) {
+			log.error("bigquery io error",e);
+			throw new RuntimeIOException(e);
+		} catch (InterruptedException e) {
+			log.error("bigquery job error",e);
+			throw new RuntimeException(e);
+		}
+	}
+	
+	public Schema extractTableToGcs(Bigquery bigquery, PluginTask task){
+		try {
     		// extract table and get schema
     		Schema schema = BigqueryExportUtils.extractWorkTable(bigquery, task);
-    		
     		return schema;
 		} catch (IOException e) {
 			log.error("bigquery io error",e);
@@ -231,11 +266,12 @@ public class BigqueryExportGcsFileInputPlugin
 			log.error("bigquery job error",e);
 			throw new RuntimeException(e);
 		}
-    }
-    // usually, you have an method to create list of files
+	}
+
+	// usually, you have an method to create list of files
     List<String> listFilesOfGcs(PluginTask task)
     {
-    	log.info("get file list in to gcs of ... {}.{} -> gs://{}/{}", task.getDataset(), task.getWorkTable(),task.getGcsBucket(),task.getGcsBlobNamePrefix());
+    	log.info("get file list in to gcs of ... {}.{} -> gs://{}/{}", task.getWorkDataset(), task.getWorkTable(),task.getGcsBucket(),task.getGcsBlobNamePrefix());
     	
     	try {
 			return BigqueryExportUtils.getFileListFromGcs(task);
@@ -298,8 +334,7 @@ public class BigqueryExportGcsFileInputPlugin
     			p.toFile().delete();
     	    	
     			if(task.getCleanupGcsTempFile()){
-    				//TODO : delete temp file in gcs
-    				log.info("delete temp gcs file... {} ... not now... ", file);
+    				BigqueryExportUtils.removeTempGcsFiles(task, file);
     			}
     			
     			//		

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java

@@ -25,6 +25,7 @@ import org.slf4j.Logger;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.api.client.json.JsonFactory;
@@ -52,6 +53,8 @@ import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 
+import io.airlift.slice.RuntimeIOException;
+
 /**
  * 
  * 
@@ -108,7 +111,9 @@ public class BigqueryExportUtils
 
     public static void executeQueryToDestinationWorkTable(Bigquery bigquery, PluginTask task) throws IOException, InterruptedException {
 
-		log.info("extract query result {} => {}.{} ",task.getQuery().get(), task.getWorkDataset(), task.getWorkTable());
+    	log.info("execute Query to Table ");
+    	log.info("# Query # {}",task.getQuery().get());
+		log.info("# Table # {}.{} ",task.getWorkDataset(), task.getWorkTable());
 		
 	    JobConfigurationQuery queryConfig = new JobConfigurationQuery();
 	    queryConfig.setQuery(task.getQuery().get());
@@ -130,8 +135,7 @@ public class BigqueryExportUtils
 	    JobReference jobRef = jobRes.getJobReference();
 		String jobId = jobRef.getJobId();
 
-		log.info("query to Table jobId : {}",jobId);
-		log.info("waiting for job end....... ");
+		log.info("query to Table jobId : {} : waiting for job end...",jobId);
 		
 		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getBigqueryJobWaitingSecond().get());
 		
@@ -158,10 +162,15 @@ public class BigqueryExportUtils
      * @throws IOException 
      * @throws FileNotFoundException 
      */
-    public static Bigquery newBigqueryClient(PluginTask task) throws FileNotFoundException, IOException{
+    public static Bigquery newBigqueryClient(PluginTask task){
     	log.debug("# Starting Google BigQuery API ... ");
-    	GoogleCredentialSet set = googleCredential(task);
-		return new Bigquery.Builder(set.transport, set.jsonFactory, set.googleCredential).setApplicationName("embulk-input-bigquey-export-gcs").build();
+		try {
+			GoogleCredentialSet set = googleCredential(task);
+			return new Bigquery.Builder(set.transport, set.jsonFactory, set.googleCredential).setApplicationName("embulk-input-bigquey-export-gcs").build();
+		} catch (Exception e) {
+			throw new RuntimeException("bigquery connect fail",e);
+		}
+		
     }
 
     public static Storage newGcsClient(PluginTask task) throws FileNotFoundException, IOException{
@@ -244,7 +253,7 @@ public class BigqueryExportUtils
     	return builder.build();
     }
     
-    public static void initWorkTableWithExecuteQuery(Bigquery bigquery, PluginTask task) throws FileNotFoundException, IOException, InterruptedException{
+    public static PHASE initTask(PluginTask task) {
 
 		if(task.getQuery().isPresent()){
 			task.setWorkId(generateTempTableName(task.getQuery().get()));
@@ -259,17 +268,18 @@ public class BigqueryExportUtils
 			// actual target table setting
 			task.setWorkDataset(task.getTempDataset().get());
 			task.setWorkTable(task.getTempTable().get());
-
-			// call google api
-			executeQueryToDestinationWorkTable(bigquery, task);
 			
+			return PHASE.QUERY;
 		}else if(task.getTable().isPresent() && task.getDataset().isPresent()){
 			task.setWorkId(generateTempTableName(null, task.getTable().get()));
 			// actual target table setting			
 			task.setWorkDataset(task.getDataset().get());
 			task.setWorkTable(task.getTable().get());
+			
+			return PHASE.TABLE;
+			
 		}else{
-			throw new IOException("please set config file [dataset]+[table] or [query]");
+			throw new RuntimeException("please set config file [dataset]+[table] or [query]");
 		}
     }
     
@@ -301,7 +311,10 @@ public class BigqueryExportUtils
 		log.info("extract jobId : {}",jobId);
 		log.debug("waiting for job end....... ");
 		
-		waitForJob(bigquery, task.getProject(), jobId, task.getBigqueryJobWaitingSecond().get());
+		Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getBigqueryJobWaitingSecond().get());
+		
+		log.info("table extract result : {}",lastJob.toPrettyString());
+		
 		return embulkSchema;
     }
 
@@ -437,14 +450,28 @@ public class BigqueryExportUtils
 		}
     }
     
+    public static void removeGcsFilesBeforeExecuting(PluginTask task){
+		try {
+			Storage gcs = BigqueryExportUtils.newGcsClient(task);
+			gcs.objects().delete(task.getGcsBucket(), task.getGcsBlobNamePrefix()).execute();
+		} catch (GoogleJsonResponseException e) {
+			if(e.getStatusCode() == 404){
+				log.info("file not found in gs://{}/{} :: it's ok ",task.getGcsBucket(), task.getGcsBlobNamePrefix());
+			}else{
+				throw new RuntimeException("# Remove GCS files gs://" + task.getGcsBucket() + "/" + task.getGcsBlobNamePrefix(),e);	
+			}
+		} catch (Exception e) {
+			throw new RuntimeException("# Remove GCS files gs://" + task.getGcsBucket() + "/" + task.getGcsBlobNamePrefix(),e);
+		}
+    }
+    
     public static void removeTempGcsFiles(PluginTask task, String file){
 		try {
 			Storage gcs = BigqueryExportUtils.newGcsClient(task);
-			Bucket bucket = gcs.buckets().get(task.getGcsBucket()).execute();
-			List<String> fileList = task.getFiles();
-			
+			log.info("delete finish file gs://{}{}", task.getGcsBucket(), file);
+			gcs.objects().delete(task.getGcsBucket(), file).execute();
 		} catch (Exception e) {
-			log.error("# Remove temp table FAIL : " + task.getTempDataset().orNull() +  "." + task.getTempTable().orNull(),e);
+			log.error("# Remove temp gcs file FAIL : " + file,e);
 		}
     }
 }

data/src/main/java/org/embulk/input/bigquery_export_gcs/PHASE.java

@@ -0,0 +1,7 @@
+package org.embulk.input.bigquery_export_gcs;
+
+public enum PHASE {
+	QUERY,
+	TABLE,
+	GCS
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-bigquery_extract_files
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - jo8937
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-20 00:00:00.000000000 Z
+date: 2017-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -59,12 +59,13 @@ files:
 - lib/embulk/input/bigquery_extract_files.rb
 - src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java
 - src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java
+- src/main/java/org/embulk/input/bigquery_export_gcs/PHASE.java
 - src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java
 - src/test/java/org/embulk/input/bigquery_export_gcs/TestPluginFunctions.java
 - src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-bigquery_extract_files-0.0.4.jar
+- classpath/embulk-input-bigquery_extract_files-0.0.5.jar
 - classpath/google-api-client-1.23.0.jar
 - classpath/google-api-services-bigquery-v2-rev363-1.23.0.jar
 - classpath/google-api-services-storage-v1-rev59-1.21.0.jar