embulk-input-bigquery_extract_files 0.0.10 → 0.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -0
  3. data/build.gradle +5 -4
  4. data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java +4 -0
  5. data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java +5 -5
  6. data/src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java +4 -2
  7. metadata +14 -12
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9fd93a8f3a7c23a8e9abe77ce5ee506be8616257
4
- data.tar.gz: 3658a9965a2b6e08e374bf311840471dc29ba1bc
3
+ metadata.gz: bf54095d7c9086ce7200d2172be94c984a14450f
4
+ data.tar.gz: 779e6cf380358f43f92d308c75f32597b38dee28
5
5
  SHA512:
6
- metadata.gz: 9ffe3c12eec23dbde7f32e7c4be48b2c7a65be282adbed14d54e1d2bb7e057fe77701559144887bef3bda376a9ca3741fcd22298e51c1765c79eae15b15faa02
7
- data.tar.gz: 7fcf8e3fd0ec1536feee4c43bb4e32aec909ff1cb14142d574e7351aacb09bd3fff2802ff4c33252b14e20bae4d6b377c6d473a874386931e7a6f4e9fef60690
6
+ metadata.gz: b7d020bf8022e232207cd9ec36c202868c2efe0aaeb85cece3625bde41ea7bebce6ce973ec2758cf7ecb4093a282c9a6f06c58637a99acfe84f7867cdcfcb965
7
+ data.tar.gz: a3503f60a0e9a6cec0f5e48ba62eb16fefa7e576c0db0d71354046314bae9e72894f541309d48aa92c8c103dd1c0561c24905d0c3acbcd3684d83bbeb1347d25
data/README.md CHANGED
@@ -42,6 +42,7 @@ embulk gem install embulk-input-bigquery_extract_files
42
42
 
43
43
  - **project**: Google Cloud Platform (gcp) project id (string, required)
44
44
  - **json_keyfile**: gcp service account's private key with json (string, required)
45
+ - **location**: location of bigquery dataset and temp_dataset. see : https://cloud.google.com/bigquery/docs/locations (Optional) (string, default: `US`)
45
46
  - **gcs_uri**: bigquery result saved uri. bucket and path names parsed from this uri. (string, required)
46
47
  - **temp_local_path**: extract files download directory in local machine (string, required)
47
48
 
@@ -71,6 +72,8 @@ embulk gem install embulk-input-bigquery_extract_files
71
72
 
72
73
  - **cleanup_local_temp_files**: delete all file in local temp dir (Optional) (string, default: `true`)
73
74
 
75
+ - **direct_download_enabled**: gcs download option. see : https://developers.google.com/api-client-library/java/google-api-java-client/reference/1.19.1/com/google/api/client/googleapis/media/MediaHttpDownloader#setDirectDownloadEnabled(boolean) (Optional) (string, default: `false`)
76
+
74
77
  - **decoders**: embulk java-file-input plugin's default attribute. see : http://www.embulk.org/docs/built-in.html#gzip-decoder-plugin
75
78
  - **parser**: embulk java-file-input plugin's default .attribute see : http://www.embulk.org/docs/built-in.html#csv-parser-plugin
76
79
 
data/build.gradle CHANGED
@@ -13,7 +13,7 @@ configurations {
13
13
  provided
14
14
  }
15
15
 
16
- version = "0.0.10"
16
+ version = "0.0.11"
17
17
 
18
18
  sourceCompatibility = 1.7
19
19
  targetCompatibility = 1.7
@@ -22,11 +22,12 @@ dependencies {
22
22
  compile ("org.embulk:embulk-core:0.8.36")
23
23
  provided("org.embulk:embulk-core:0.8.36")
24
24
 
25
- compile "com.google.http-client:google-http-client-jackson2:1.21.0"
25
+ compile "com.google.guava:guava:16+"
26
+ compile "com.google.http-client:google-http-client-jackson2:1.28.0"
26
27
  //compile ('com.google.apis:google-api-services-bigquery:v2-rev363-1.23.0') {exclude module: "guava-jdk5"}
27
- compile ('com.google.apis:google-api-services-bigquery:v2-rev402-1.25.0') {exclude module: "guava-jdk5"}
28
+ compile ('com.google.apis:google-api-services-bigquery:v2-rev429-1.25.0') {exclude module: "guava-jdk5"}
28
29
  //compile ("com.google.apis:google-api-services-storage:v1-rev59-1.21.0") {exclude module: "guava-jdk5"}
29
- compile ("com.google.apis:google-api-services-storage:v1-rev136-1.25.0") {exclude module: "guava-jdk5"}
30
+ compile ("com.google.apis:google-api-services-storage:v1-rev150-1.25.0") {exclude module: "guava-jdk5"}
30
31
 
31
32
  testCompile "junit:junit:4.+"
32
33
  testCompile "org.embulk:embulk-core:0.8.36:tests"
data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java CHANGED
@@ -60,6 +60,10 @@ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
60
60
  @Config("json_keyfile")
61
61
  public String getJsonKeyfile();
62
62
 
63
+ @Config("location")
64
+ @ConfigDefault("\"US\"")
65
+ public Optional<String> getLocation();
66
+
63
67
  @Config("dataset")
64
68
  @ConfigDefault("null")
65
69
  public Optional<String> getDataset();
data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java CHANGED
@@ -33,7 +33,7 @@ import com.google.api.client.http.HttpTransport;
33
33
  import com.google.api.client.http.javanet.NetHttpTransport;
34
34
  import com.google.api.client.json.JsonFactory;
35
35
  import com.google.api.client.json.jackson2.JacksonFactory;
36
- import com.google.api.client.repackaged.com.google.common.base.Strings;
36
+ import com.google.common.base.Strings;
37
37
  import com.google.api.services.bigquery.Bigquery;
38
38
  import com.google.api.services.bigquery.Bigquery.Jobs.Insert;
39
39
  import com.google.api.services.bigquery.Bigquery.Tabledata;
@@ -143,7 +143,7 @@ public class BigqueryExportUtils
143
143
 
144
144
  log.info("query to Table jobId : {} : waiting for job end...",jobId);
145
145
 
146
- Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getBigqueryJobWaitingSecond().get());
146
+ Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get());
147
147
 
148
148
  log.debug("waiting for job end....... {}", lastJob.toPrettyString());
149
149
  }
@@ -339,21 +339,21 @@ public class BigqueryExportUtils
339
339
  log.info("extract jobId : {}",jobId);
340
340
  log.debug("waiting for job end....... ");
341
341
 
342
- Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getBigqueryJobWaitingSecond().get());
342
+ Job lastJob = waitForJob(bigquery, task.getProject(), jobId, task.getLocation().get(), task.getBigqueryJobWaitingSecond().get());
343
343
 
344
344
  log.info("table extract result : {}",lastJob.toPrettyString());
345
345
 
346
346
  return embulkSchema;
347
347
  }
348
348
 
349
- public static Job waitForJob(Bigquery bigquery, String project, String jobId, int bigqueryJobWaitingSecond) throws IOException, InterruptedException{
349
+ public static Job waitForJob(Bigquery bigquery, String project, String jobId, String location, int bigqueryJobWaitingSecond) throws IOException, InterruptedException{
350
350
  int maxAttempts = bigqueryJobWaitingSecond;
351
351
  int initialRetryDelay = 1000; // ms
352
352
  Job pollingJob = null;
353
353
  log.info("waiting for job end : {}",jobId);
354
354
  int tryCnt = 0;
355
355
  for (tryCnt=0; tryCnt < maxAttempts; tryCnt++){
356
- pollingJob = bigquery.jobs().get(project, jobId).execute();
356
+ pollingJob = bigquery.jobs().get(project, jobId).setLocation(location).execute();
357
357
  String state = pollingJob.getStatus().getState();
358
358
  log.debug("Job Status {} : {}",jobId, state);
359
359
 
data/src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java CHANGED
@@ -20,8 +20,10 @@ public class TestGoogleCloudAccessData extends UnitTestInitializer
20
20
  plugin.executeBigqueryApi(task);
21
21
 
22
22
  InputStream ins = BigqueryExportUtils.openInputStream(task, task.getFiles().get(0));
23
-
24
- log.info("file size : {}",org.apache.commons.compress.utils.IOUtils.toByteArray(ins).length);
23
+
24
+ byte[] b = org.apache.commons.compress.utils.IOUtils.toByteArray(ins);
25
+ log.info("file size : {}",b.length);
26
+ log.info("file cont : {}",new String(b));
25
27
  }
26
28
 
27
29
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-bigquery_extract_files
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.10
4
+ version: 0.0.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - jo8937
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-08-28 00:00:00.000000000 Z
11
+ date: 2019-03-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -63,21 +63,23 @@ files:
63
63
  - src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java
64
64
  - src/test/java/org/embulk/input/bigquery_export_gcs/TestPluginFunctions.java
65
65
  - src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java
66
- - classpath/commons-codec-1.10.jar
67
- - classpath/commons-logging-1.2.jar
68
- - classpath/embulk-input-bigquery_extract_files-0.0.10.jar
66
+ - classpath/animal-sniffer-annotations-1.14.jar
67
+ - classpath/checker-compat-qual-2.5.2.jar
68
+ - classpath/embulk-input-bigquery_extract_files-0.0.11.jar
69
+ - classpath/error_prone_annotations-2.1.3.jar
69
70
  - classpath/google-api-client-1.25.0.jar
70
- - classpath/google-api-services-bigquery-v2-rev402-1.25.0.jar
71
- - classpath/google-api-services-storage-v1-rev136-1.25.0.jar
72
- - classpath/google-http-client-1.25.0.jar
73
- - classpath/google-http-client-jackson2-1.25.0.jar
71
+ - classpath/google-api-services-bigquery-v2-rev429-1.25.0.jar
72
+ - classpath/google-api-services-storage-v1-rev150-1.25.0.jar
73
+ - classpath/google-http-client-1.28.0.jar
74
+ - classpath/google-http-client-jackson2-1.28.0.jar
74
75
  - classpath/google-oauth-client-1.25.0.jar
75
- - classpath/guava-20.0.jar
76
- - classpath/httpclient-4.5.5.jar
77
- - classpath/httpcore-4.4.9.jar
76
+ - classpath/grpc-context-1.14.0.jar
77
+ - classpath/guava-26.0-android.jar
78
78
  - classpath/j2objc-annotations-1.1.jar
79
79
  - classpath/jackson-core-2.9.6.jar
80
80
  - classpath/jsr305-3.0.2.jar
81
+ - classpath/opencensus-api-0.18.0.jar
82
+ - classpath/opencensus-contrib-http-util-0.18.0.jar
81
83
  homepage: https://github.com/jo8937/embulk-input-bigquery_extract_files
82
84
  licenses:
83
85
  - MIT