embulk-input-bigquery_extract_files 0.0.7 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +16 -16
  3. data/LICENSE.txt +21 -21
  4. data/README.md +168 -144
  5. data/build.gradle +104 -102
  6. data/config.yml +31 -31
  7. data/config/checkstyle/checkstyle.xml +128 -128
  8. data/config/checkstyle/default.xml +108 -108
  9. data/gradle/wrapper/gradle-wrapper.properties +5 -5
  10. data/gradlew +172 -172
  11. data/lib/embulk/input/bigquery_extract_files.rb +3 -3
  12. data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java +378 -383
  13. data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java +519 -495
  14. data/src/main/java/org/embulk/input/bigquery_export_gcs/PHASE.java +7 -7
  15. data/src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java +33 -33
  16. data/src/test/java/org/embulk/input/bigquery_export_gcs/TestPluginFunctions.java +56 -56
  17. data/src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java +96 -96
  18. metadata +18 -15
data/src/main/java/org/embulk/input/bigquery_export_gcs/PHASE.java CHANGED
@@ -1,7 +1,7 @@
1
- package org.embulk.input.bigquery_export_gcs;
2
-
3
- public enum PHASE {
4
- QUERY,
5
- TABLE,
6
- GCS
7
- }
1
+ package org.embulk.input.bigquery_export_gcs;
2
+
3
+ public enum PHASE {
4
+ QUERY,
5
+ TABLE,
6
+ GCS
7
+ }
data/src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java CHANGED
@@ -1,33 +1,33 @@
1
- package org.embulk.input.bigquery_export_gcs;
2
-
3
- import java.io.FileNotFoundException;
4
- import java.io.IOException;
5
- import java.io.InputStream;
6
-
7
- import org.junit.Test;
8
- import org.slf4j.Logger;
9
- import org.slf4j.LoggerFactory;
10
-
11
- public class TestGoogleCloudAccessData extends UnitTestInitializer
12
- {
13
- private static final Logger log = LoggerFactory.getLogger(TestGoogleCloudAccessData.class);
14
-
15
- @Test
16
- public void envTest(){
17
- log.info("{}",System.getenv("GCP_PROJECT"));
18
- }
19
-
20
- @Test
21
- public void testGcsInputStreamOpen() throws FileNotFoundException, IOException
22
- {
23
- BigqueryExportGcsFileInputPlugin.PluginTask task = config.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
24
-
25
- plugin.executeBigqueryApi(task);
26
-
27
- InputStream ins = BigqueryExportUtils.openInputStream(task, task.getFiles().get(0));
28
-
29
- log.info("file size : {}",org.apache.commons.compress.utils.IOUtils.toByteArray(ins).length);
30
- }
31
-
32
-
33
- }
1
+ package org.embulk.input.bigquery_export_gcs;
2
+
3
+ import java.io.FileNotFoundException;
4
+ import java.io.IOException;
5
+ import java.io.InputStream;
6
+
7
+ import org.junit.Test;
8
+ import org.slf4j.Logger;
9
+ import org.slf4j.LoggerFactory;
10
+
11
+ public class TestGoogleCloudAccessData extends UnitTestInitializer
12
+ {
13
+ private static final Logger log = LoggerFactory.getLogger(TestGoogleCloudAccessData.class);
14
+
15
+ @Test
16
+ public void envTest(){
17
+ log.info("{}",System.getenv("GCP_PROJECT"));
18
+ }
19
+
20
+ @Test
21
+ public void testGcsInputStreamOpen() throws FileNotFoundException, IOException
22
+ {
23
+ BigqueryExportGcsFileInputPlugin.PluginTask task = config.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
24
+
25
+ plugin.executeBigqueryApi(task);
26
+
27
+ InputStream ins = BigqueryExportUtils.openInputStream(task, task.getFiles().get(0));
28
+
29
+ log.info("file size : {}",org.apache.commons.compress.utils.IOUtils.toByteArray(ins).length);
30
+ }
31
+
32
+
33
+ }
data/src/test/java/org/embulk/input/bigquery_export_gcs/TestPluginFunctions.java CHANGED
@@ -1,56 +1,56 @@
1
- package org.embulk.input.bigquery_export_gcs;
2
-
3
- import static org.junit.Assert.assertEquals;
4
- import static org.junit.Assert.assertNotNull;
5
- import static org.junit.Assume.assumeNotNull;
6
-
7
- import java.io.FileNotFoundException;
8
- import java.io.IOException;
9
- import java.io.InputStream;
10
- import java.util.List;
11
-
12
- import org.embulk.EmbulkTestRuntime;
13
- import org.embulk.config.ConfigSource;
14
- import org.embulk.spi.Exec;
15
- import org.embulk.spi.FileInputRunner;
16
- import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
17
- import org.junit.Before;
18
- import org.junit.BeforeClass;
19
- import org.junit.Rule;
20
- import org.junit.Test;
21
- import org.slf4j.Logger;
22
- import org.slf4j.LoggerFactory;
23
-
24
- public class TestPluginFunctions extends UnitTestInitializer
25
- {
26
- private static final Logger log = LoggerFactory.getLogger(TestPluginFunctions.class);
27
-
28
- @Test
29
- public void regexTest(){
30
- testTableName("select * from aaa.bbb","bbb");
31
- testTableName("select * from aaa.bbb.ccc","ccc");
32
- testTableName("select * from [aaa.bbb]","bbb");
33
- testTableName("select * from aaa.bbb$20171123","bbb");
34
- testTableName("select * from aaa.t_b_b_b","t_b_b_b");
35
- }
36
-
37
- public void testTableName(String query, String expect){
38
- String word = BigqueryExportUtils.parseQueryToBaseTableName(query);
39
- log.info("{}", word );
40
- assertEquals(word, expect);
41
- }
42
-
43
- @Test
44
- public void testParseGcsUrl(){
45
- ConfigSource c = config.deepCopy();
46
- c.set("gcs_uri", "gs://aaa/bbb/ccc_*");
47
-
48
- BigqueryExportGcsFileInputPlugin.PluginTask task = c.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
49
-
50
- BigqueryExportUtils.parseGcsUri(task);
51
-
52
- assertEquals("", "aaa", task.getGcsBucket());
53
- assertEquals("", "bbb/ccc_", task.getGcsBlobNamePrefix());
54
- }
55
-
56
- }
1
+ package org.embulk.input.bigquery_export_gcs;
2
+
3
+ import static org.junit.Assert.assertEquals;
4
+ import static org.junit.Assert.assertNotNull;
5
+ import static org.junit.Assume.assumeNotNull;
6
+
7
+ import java.io.FileNotFoundException;
8
+ import java.io.IOException;
9
+ import java.io.InputStream;
10
+ import java.util.List;
11
+
12
+ import org.embulk.EmbulkTestRuntime;
13
+ import org.embulk.config.ConfigSource;
14
+ import org.embulk.spi.Exec;
15
+ import org.embulk.spi.FileInputRunner;
16
+ import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
17
+ import org.junit.Before;
18
+ import org.junit.BeforeClass;
19
+ import org.junit.Rule;
20
+ import org.junit.Test;
21
+ import org.slf4j.Logger;
22
+ import org.slf4j.LoggerFactory;
23
+
24
+ public class TestPluginFunctions extends UnitTestInitializer
25
+ {
26
+ private static final Logger log = LoggerFactory.getLogger(TestPluginFunctions.class);
27
+
28
+ @Test
29
+ public void regexTest(){
30
+ testTableName("select * from aaa.bbb","bbb");
31
+ testTableName("select * from aaa.bbb.ccc","ccc");
32
+ testTableName("select * from [aaa.bbb]","bbb");
33
+ testTableName("select * from aaa.bbb$20171123","bbb");
34
+ testTableName("select * from aaa.t_b_b_b","t_b_b_b");
35
+ }
36
+
37
+ public void testTableName(String query, String expect){
38
+ String word = BigqueryExportUtils.parseQueryToBaseTableName(query);
39
+ log.info("{}", word );
40
+ assertEquals(word, expect);
41
+ }
42
+
43
+ @Test
44
+ public void testParseGcsUrl(){
45
+ ConfigSource c = config.deepCopy();
46
+ c.set("gcs_uri", "gs://aaa/bbb/ccc_*");
47
+
48
+ BigqueryExportGcsFileInputPlugin.PluginTask task = c.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
49
+
50
+ BigqueryExportUtils.parseGcsUri(task);
51
+
52
+ assertEquals("", "aaa", task.getGcsBucket());
53
+ assertEquals("", "bbb/ccc_", task.getGcsBlobNamePrefix());
54
+ }
55
+
56
+ }
data/src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java CHANGED
@@ -1,96 +1,96 @@
1
- package org.embulk.input.bigquery_export_gcs;
2
-
3
- import static org.junit.Assume.assumeNotNull;
4
-
5
- import java.io.File;
6
- import java.io.IOException;
7
-
8
- import org.embulk.EmbulkTestRuntime;
9
- import org.embulk.config.ConfigLoader;
10
- import org.embulk.config.ConfigSource;
11
- import org.embulk.spi.Exec;
12
- import org.embulk.spi.FileInputRunner;
13
- import org.embulk.spi.SchemaConfig;
14
- import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
15
- import org.junit.Before;
16
- import org.junit.BeforeClass;
17
- import org.junit.Rule;
18
- import org.junit.Test;
19
- import org.slf4j.Logger;
20
- import org.slf4j.LoggerFactory;
21
-
22
- import com.fasterxml.jackson.databind.JsonNode;
23
-
24
- public class UnitTestInitializer
25
- {
26
- private static final Logger log = LoggerFactory.getLogger(UnitTestInitializer.class);
27
-
28
- private static String TEST_EMBULK_CUSTOM_CONFIG_PATH;
29
-
30
- /*
31
- * This test case requires environment variables
32
- * GCP_EMAIL
33
- * GCP_P12_KEYFILE
34
- * GCP_JSON_KEYFILE
35
- * GCP_BUCKET
36
- */
37
- @BeforeClass
38
- public static void initializeConstantVariables()
39
- {
40
- TEST_EMBULK_CUSTOM_CONFIG_PATH = "local_config.yml"; //System.getenv("TEST_EMBULK_CUSTOM_CONFIG_PATH");
41
- }
42
-
43
- @Rule
44
- public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
45
- protected ConfigSource config;
46
- protected BigqueryExportGcsFileInputPlugin plugin;
47
- protected FileInputRunner runner;
48
- protected MockPageOutput output;
49
-
50
- /**
51
- * @throws IOException
52
- *
53
- */
54
- @Before
55
- public void createResources() throws IOException
56
- {
57
- File f =new File(TEST_EMBULK_CUSTOM_CONFIG_PATH);
58
- if(f.exists()){
59
- log.info("Test env load from file : {}",f);
60
- config = new ConfigLoader(Exec.session().getModelManager()).fromYamlFile(f).getNested("in");
61
- log.info("values : {}",config.toString());
62
- }else{
63
- log.error("test config file NOT FOUND ! :: {}",f);
64
- log.info("load test env from system env ... ");
65
- config = Exec.newConfigSource();
66
- config.set("project", System.getenv("embulk_config_project"));
67
- config.set("json_keyfile", System.getenv("embulk_config_json_keyfile"));
68
- config.set("dataset", System.getenv("embulk_config_dataset"));
69
- config.set("query", System.getenv("embulk_config_query"));
70
- config.set("gcs_uri", System.getenv("embulk_config_gcs_uri"));
71
- config.set("temp_dataset", System.getenv("embulk_config_temp_dataset"));
72
- config.set("temp_local_path", System.getenv("embulk_config_temp_local_path"));
73
- }
74
-
75
- plugin = new BigqueryExportGcsFileInputPlugin();
76
- runner = new FileInputRunner(runtime.getInstance(BigqueryExportGcsFileInputPlugin.class));
77
- output = new MockPageOutput();
78
-
79
- assumeNotNull(
80
- config.get(String.class, "project"),
81
- config.get(String.class, "json_keyfile"),
82
- config.get(String.class, "gcs_uri"),
83
- config.get(String.class, "temp_local_path")
84
- );
85
- }
86
-
87
- @Test
88
- public void testInitTask(){
89
- BigqueryExportGcsFileInputPlugin.PluginTask task = config.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
90
-
91
- log.info("{}", BigqueryExportUtils.toPrettyString(task.dump().deepCopy().getObjectNode()) );
92
- }
93
-
94
-
95
-
96
- }
1
+ package org.embulk.input.bigquery_export_gcs;
2
+
3
+ import static org.junit.Assume.assumeNotNull;
4
+
5
+ import java.io.File;
6
+ import java.io.IOException;
7
+
8
+ import org.embulk.EmbulkTestRuntime;
9
+ import org.embulk.config.ConfigLoader;
10
+ import org.embulk.config.ConfigSource;
11
+ import org.embulk.spi.Exec;
12
+ import org.embulk.spi.FileInputRunner;
13
+ import org.embulk.spi.SchemaConfig;
14
+ import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
15
+ import org.junit.Before;
16
+ import org.junit.BeforeClass;
17
+ import org.junit.Rule;
18
+ import org.junit.Test;
19
+ import org.slf4j.Logger;
20
+ import org.slf4j.LoggerFactory;
21
+
22
+ import com.fasterxml.jackson.databind.JsonNode;
23
+
24
+ public class UnitTestInitializer
25
+ {
26
+ private static final Logger log = LoggerFactory.getLogger(UnitTestInitializer.class);
27
+
28
+ private static String TEST_EMBULK_CUSTOM_CONFIG_PATH;
29
+
30
+ /*
31
+ * This test case requires environment variables
32
+ * GCP_EMAIL
33
+ * GCP_P12_KEYFILE
34
+ * GCP_JSON_KEYFILE
35
+ * GCP_BUCKET
36
+ */
37
+ @BeforeClass
38
+ public static void initializeConstantVariables()
39
+ {
40
+ TEST_EMBULK_CUSTOM_CONFIG_PATH = "local_config.yml"; //System.getenv("TEST_EMBULK_CUSTOM_CONFIG_PATH");
41
+ }
42
+
43
+ @Rule
44
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
45
+ protected ConfigSource config;
46
+ protected BigqueryExportGcsFileInputPlugin plugin;
47
+ protected FileInputRunner runner;
48
+ protected MockPageOutput output;
49
+
50
+ /**
51
+ * @throws IOException
52
+ *
53
+ */
54
+ @Before
55
+ public void createResources() throws IOException
56
+ {
57
+ File f =new File(TEST_EMBULK_CUSTOM_CONFIG_PATH);
58
+ if(f.exists()){
59
+ log.info("Test env load from file : {}",f);
60
+ config = new ConfigLoader(Exec.session().getModelManager()).fromYamlFile(f).getNested("in");
61
+ log.info("values : {}",config.toString());
62
+ }else{
63
+ log.error("test config file NOT FOUND ! :: {}",f);
64
+ log.info("load test env from system env ... ");
65
+ config = Exec.newConfigSource();
66
+ config.set("project", System.getenv("embulk_config_project"));
67
+ config.set("json_keyfile", System.getenv("embulk_config_json_keyfile"));
68
+ config.set("dataset", System.getenv("embulk_config_dataset"));
69
+ config.set("query", System.getenv("embulk_config_query"));
70
+ config.set("gcs_uri", System.getenv("embulk_config_gcs_uri"));
71
+ config.set("temp_dataset", System.getenv("embulk_config_temp_dataset"));
72
+ config.set("temp_local_path", System.getenv("embulk_config_temp_local_path"));
73
+ }
74
+
75
+ plugin = new BigqueryExportGcsFileInputPlugin();
76
+ runner = new FileInputRunner(runtime.getInstance(BigqueryExportGcsFileInputPlugin.class));
77
+ output = new MockPageOutput();
78
+
79
+ assumeNotNull(
80
+ config.get(String.class, "project"),
81
+ config.get(String.class, "json_keyfile"),
82
+ config.get(String.class, "gcs_uri"),
83
+ config.get(String.class, "temp_local_path")
84
+ );
85
+ }
86
+
87
+ @Test
88
+ public void testInitTask(){
89
+ BigqueryExportGcsFileInputPlugin.PluginTask task = config.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
90
+
91
+ log.info("{}", BigqueryExportUtils.toPrettyString(task.dump().deepCopy().getObjectNode()) );
92
+ }
93
+
94
+
95
+
96
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-bigquery_extract_files
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.7
4
+ version: 0.0.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - jo8937
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-11-25 00:00:00.000000000 Z
11
+ date: 2018-08-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
38
38
  - - '>='
39
39
  - !ruby/object:Gem::Version
40
40
  version: '10.0'
41
- description: Reads files stored on Google Cloud Storage, that exportes from Bigquery query result or table
41
+ description: Reads files stored on Google Cloud Storage that extracted from bigquery table or query result
42
42
  email:
43
43
  - jo8937@gmail.com
44
44
  executables: []
@@ -63,18 +63,21 @@ files:
63
63
  - src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java
64
64
  - src/test/java/org/embulk/input/bigquery_export_gcs/TestPluginFunctions.java
65
65
  - src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java
66
- - classpath/commons-codec-1.3.jar
67
- - classpath/commons-logging-1.1.1.jar
68
- - classpath/embulk-input-bigquery_extract_files-0.0.7.jar
69
- - classpath/google-api-client-1.23.0.jar
70
- - classpath/google-api-services-bigquery-v2-rev363-1.23.0.jar
71
- - classpath/google-api-services-storage-v1-rev59-1.21.0.jar
72
- - classpath/google-http-client-1.23.0.jar
73
- - classpath/google-http-client-jackson2-1.23.0.jar
74
- - classpath/google-oauth-client-1.23.0.jar
75
- - classpath/httpclient-4.0.1.jar
76
- - classpath/httpcore-4.0.1.jar
77
- - classpath/jsr305-1.3.9.jar
66
+ - classpath/commons-codec-1.10.jar
67
+ - classpath/commons-logging-1.2.jar
68
+ - classpath/embulk-input-bigquery_extract_files-0.0.9.jar
69
+ - classpath/google-api-client-1.25.0.jar
70
+ - classpath/google-api-services-bigquery-v2-rev402-1.25.0.jar
71
+ - classpath/google-api-services-storage-v1-rev136-1.25.0.jar
72
+ - classpath/google-http-client-1.25.0.jar
73
+ - classpath/google-http-client-jackson2-1.25.0.jar
74
+ - classpath/google-oauth-client-1.25.0.jar
75
+ - classpath/guava-20.0.jar
76
+ - classpath/httpclient-4.5.5.jar
77
+ - classpath/httpcore-4.4.9.jar
78
+ - classpath/j2objc-annotations-1.1.jar
79
+ - classpath/jackson-core-2.9.6.jar
80
+ - classpath/jsr305-3.0.2.jar
78
81
  homepage: https://github.com/jo8937/embulk-input-bigquery_extract_files
79
82
  licenses:
80
83
  - MIT