embulk-input-bigquery_extract_files 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e709d68054ae241b205a52159c85d2eaec65406a
-  data.tar.gz: 88d85008b319fd53613d347dfaa101ad1a106f36
+  metadata.gz: 0728b31117097ca0eef206be33646b0e210a85aa
+  data.tar.gz: 1ce4c3bd0eab4385250bfd7a9876866cd7f4f136
 SHA512:
-  metadata.gz: 951cf84517c6ffa2785c4a164354f2e3ca96faf0ce3ded183946726a54d6d780bd96245e78befc60ec9c14793bb922a6a526b4574dc6dda67a7bb93ee201b200
-  data.tar.gz: 7321c8a02fdfe9fc9973f539806eca76c37c4548a12cd55475a36e1421f9b0d82e38941a64f61358f4ac420dab92f5fcbc3d4388be41360c16955b1cff0b2301
+  metadata.gz: 123ff54b8d18e9ecefaa1525a74792a98ec56537d5123f3e3a5180b62e92b84454fefe42813420fd4c16b5eddb1c300460918dae787b32291271edc2b28afb4c
+  data.tar.gz: 6c3872f92860a27bf42eb8ffc9ad71612c9defbd25f41a82234bb09fad62c5fb87492e6bd8bbf10d8b01ec4c928df4824fd8ffed941a7672b9f3e6b4cdcabd08

data/README.md

@@ -55,12 +55,19 @@ embulk gem install embulk-input-bigquery_extract_files
 
 - **temp_schema_file_path**: bigquery result schema file for parser. (Optional) (string, default: `null`)
 
-- **temp_schema_file_type**: default is embulk's Schema object. (Optional) (string, default: `null`)  
+- **bigquery_job_wait_second**: bigquery job waiting second. (Optional) (string, default: `600`)
+
+- **cleanup_gcs_before_executing**: delete all file in gcs temp path before process start (Optional) (string, default: `true`)
+
+- **cleanup_gcs_files**: delete all file in gcs temp path after process end (Optional) (string, default: `false`)
+
+- **cleanup_temp_table**: delete query result table after process end (Optional) (string, default: `true`)
+
+- **cleanup_local_temp_files**: delete all file in local temp dir (Optional) (string, default: `true`)
 
 - **decoders**: embulk java-file-input plugin's default attribute. see : http://www.embulk.org/docs/built-in.html#gzip-decoder-plugin
 - **parser**: embulk java-file-input plugin's default .attribute see : http://www.embulk.org/docs/built-in.html#csv-parser-plugin
 
-
 ## Example
 
 ```yaml
@@ -84,6 +91,51 @@ out:
   type: stdout
 ```
 
+### Advenced Example 
+
+#### bigquery to mysql with auto-schema 
+
+I have to batch bigquery table to mysql every day for my job.
+then, I wan'to get auto-schema for this file input plugin.
+
+- see also 
+ - https://github.com/jo8937/embulk-parser-csv_with_schema_file
+ - https://github.com/embulk/embulk-output-jdbc/tree/master/embulk-output-mysql
+
+this is my best practive for bigquery to mysql batch config. 
+
+```yaml
+in:
+  type: bigquery_extract_files
+  project: my-google-project
+  json_keyfile: /tmp/embulk/google_service_account.json
+  query: 'select * from dataset.t_nitocris'
+  temp_dataset: temp_dataset
+  gcs_uri: gs://bucket/embulktemp/t_nitocris_*
+  temp_local_path: /tmp/embulk/data
+  file_format: 'CSV'
+  compression: 'GZIP'
+  temp_schema_file_path: /tmp/embulk/schema/csv_schema_nitocris.json
+  decoders:
+  - {type: gzip}
+  parser:
+    type: csv_with_schema_file
+    default_timestamp_format: '%Y-%m-%d %H:%M:%S %z'
+    schema_path: /tmp/embulk/schema/csv_schema_nitocris.json
+out:
+  type: mysql
+  host: host
+  user: user
+  password: password
+  port: 3306
+  database: MY_DATABASE
+  table: 
+  options: {connectTimeout: 0, waitTimeout: 0, enableQueryTimeouts: false, autoReconnect: true}
+  mode: insert_direct
+  retry_limit: 60
+  retry_wait: 3000
+  batch_size: 4096000
+```
 
 ## Build
 

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.6"
+version = "0.0.7"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/config.yml

@@ -26,6 +26,7 @@ out:
   database: dbname
   table: test_table
   # https://dev.mysql.com/doc/connector-j/5.1/en/connector-j-reference-configuration-properties.html
-  options: {connectTimeout: 0, enableQueryTimeouts: false, waitTimeout: 0}
-  mode: insert_direct
-  
+  options: {connectTimeout: 0, waitTimeout: 0, enableQueryTimeouts: false, waitTimeout: 0, autoReconnect: true}
+  mode: insert
+  column_options:
+    log_id: {type: 'bigint not null PRIMARY KEY'}

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java

@@ -127,12 +127,16 @@ public class BigqueryExportGcsFileInputPlugin
         
         @Config("cleanup_gcs_files")
         @ConfigDefault("false")
-        public boolean getCleanupGcsTempFile(); 
+        public boolean getCleanupGcsTempFiles(); 
         
         @Config("cleanup_temp_table")
         @ConfigDefault("true")
         public boolean getCleanupTempTable(); 
         
+        @Config("cleanup_local_temp_files")
+        @ConfigDefault("true")
+        public boolean getCleanupLocalTempFiles();
+        
         @Config("cleanup_gcs_before_executing")
         @ConfigDefault("true")
         public boolean getCleanupGcsBeforeExcuting();
@@ -328,12 +332,13 @@ public class BigqueryExportGcsFileInputPlugin
     		if( report.isEmpty() ){
     			String file = task.getFiles().get(i);
     	    	
-    			Path p = BigqueryExportUtils.getFullPath(task,file);
+    			if(task.getCleanupLocalTempFiles()) {
+    				Path p = BigqueryExportUtils.getFullPath(task,file);
+        			log.info("delete temp file...{}",p);
+        			p.toFile().delete();	
+    			}
     			
-    			log.info("delete temp file...{}",p);
-    			p.toFile().delete();
-    	    	
-    			if(task.getCleanupGcsTempFile()){
+    			if(task.getCleanupGcsTempFiles()){
     				BigqueryExportUtils.removeTempGcsFiles(task, file);
     			}
     			

data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java

@@ -486,7 +486,7 @@ public class BigqueryExportUtils
     public static void removeTempGcsFiles(PluginTask task, String file){
 		try {
 			Storage gcs = BigqueryExportUtils.newGcsClient(task);
-			log.info("delete finish file gs://{}{}", task.getGcsBucket(), file);
+			log.info("delete finish file gs://{}/{}", task.getGcsBucket(), file);
 			gcs.objects().delete(task.getGcsBucket(), file).execute();
 		} catch (Exception e) {
 			log.error("# Remove temp gcs file FAIL : " + file,e);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-bigquery_extract_files
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - jo8937
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-21 00:00:00.000000000 Z
+date: 2017-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,7 +65,7 @@ files:
 - src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-bigquery_extract_files-0.0.6.jar
+- classpath/embulk-input-bigquery_extract_files-0.0.7.jar
 - classpath/google-api-client-1.23.0.jar
 - classpath/google-api-services-bigquery-v2-rev363-1.23.0.jar
 - classpath/google-api-services-storage-v1-rev59-1.21.0.jar