embulk-input-bigquery_extract_files 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85c5df6d2dabf2727e267cabd33a2fd163e22942
-  data.tar.gz: 7b24ecc581b1b5c789ad4f33ab73303bc7c6ca8d
+  metadata.gz: e709d68054ae241b205a52159c85d2eaec65406a
+  data.tar.gz: 88d85008b319fd53613d347dfaa101ad1a106f36
 SHA512:
-  metadata.gz: 7b868edc052a4eec8cdd57fc5ae5899d25bb7fa65ab1289785e6c8e6f05e7d280a79e43ea608e31d9c883fc72776ea91b1a7cc1b381a0f4d898c651ebc203fae
-  data.tar.gz: 61c5c9616ca9b96ab7587144b25a16b7bdc59e34ebf9b6f38def677e080c8b7f5df26b75b725009df18cd35a96434d224a8644bba88510af7d307d429f1e0a54
+  metadata.gz: 951cf84517c6ffa2785c4a164354f2e3ca96faf0ce3ded183946726a54d6d780bd96245e78befc60ec9c14793bb922a6a526b4574dc6dda67a7bb93ee201b200
+  data.tar.gz: 7321c8a02fdfe9fc9973f539806eca76c37c4548a12cd55475a36e1421f9b0d82e38941a64f61358f4ac420dab92f5fcbc3d4388be41360c16955b1cff0b2301

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.5"
+version = "0.0.6"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java

@@ -212,7 +212,7 @@ public class BigqueryExportGcsFileInputPlugin
 		}
 
         PHASE phase = BigqueryExportUtils.initTask(task);
-        log.info("Configuration : {}",task.toString());
+        log.info("[0] Settings : {}", BigqueryExportUtils.toPrettyString(task.dump().deepCopy().getObjectNode()) );
         
         Bigquery bigquery = BigqueryExportUtils.newBigqueryClient(task);
        

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java

@@ -215,6 +215,10 @@ public class BigqueryExportUtils
     	
         do {
           objects = listRequest.execute();
+          if(objects.getItems() == null){
+        	  log.error("file not found in gs://{}/{}",bucket,blobName);
+        	  return builder.build();
+          }
           for(StorageObject obj : objects.getItems()){
         	  builder.add(obj.getName());  
           }
@@ -430,9 +434,18 @@ public class BigqueryExportUtils
 			log.error("error when create schema json",e);
 			return null;
 		}
-		
     		//for(Column col : schema.getColumns()) {
-    		
+    }
+    
+    public static String toPrettyString(Object obj){
+    	try {
+    		ObjectMapper mapper = new ObjectMapper();
+			String str = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(obj);
+			return str;
+		} catch (Exception e) {
+			log.error("JSON format error",e);
+			return java.util.Objects.toString(obj);
+		}
     }
     
     /**
@@ -452,8 +465,13 @@ public class BigqueryExportUtils
     
     public static void removeGcsFilesBeforeExecuting(PluginTask task){
 		try {
+			log.info("start cleanup gs://{}/{} ... ",task.getGcsBucket(), task.getGcsBlobNamePrefix());
 			Storage gcs = BigqueryExportUtils.newGcsClient(task);
-			gcs.objects().delete(task.getGcsBucket(), task.getGcsBlobNamePrefix()).execute();
+			List<String> fileList = getFileListFromGcs(gcs, task.getGcsBucket(), task.getGcsBlobNamePrefix());
+			for(String f : fileList){
+				log.info("cleanup gs://{}/{} ... ",task.getGcsBucket(), f);
+				gcs.objects().delete(task.getGcsBucket(), f).execute();	
+			}
 		} catch (GoogleJsonResponseException e) {
 			if(e.getStatusCode() == 404){
 				log.info("file not found in gs://{}/{} :: it's ok ",task.getGcsBucket(), task.getGcsBlobNamePrefix());

data/src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java

@@ -8,17 +8,19 @@ import java.io.IOException;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.DataSourceImpl;
-import org.embulk.input.bigquery_export_gcs.BigqueryExportGcsFileInputPlugin.PluginTask;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.SchemaConfig;
 import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
+import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.fasterxml.jackson.databind.JsonNode;
+
 public class UnitTestInitializer
 {
 	private static final Logger log = LoggerFactory.getLogger(UnitTestInitializer.class);
@@ -81,6 +83,14 @@ public class UnitTestInitializer
                       config.get(String.class, "temp_local_path")
                       );
     }
+    
+    @Test
+    public void testInitTask(){
+    	BigqueryExportGcsFileInputPlugin.PluginTask task = config.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
+    	
+    	log.info("{}", BigqueryExportUtils.toPrettyString(task.dump().deepCopy().getObjectNode()) );
+    }
+    
 
 
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-bigquery_extract_files
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - jo8937
@@ -65,7 +65,7 @@ files:
 - src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-bigquery_extract_files-0.0.5.jar
+- classpath/embulk-input-bigquery_extract_files-0.0.6.jar
 - classpath/google-api-client-1.23.0.jar
 - classpath/google-api-services-bigquery-v2-rev363-1.23.0.jar
 - classpath/google-api-services-storage-v1-rev59-1.21.0.jar