embulk-input-bigquery_extract_files 0.0.7 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +16 -16
  3. data/LICENSE.txt +21 -21
  4. data/README.md +168 -144
  5. data/build.gradle +104 -102
  6. data/config.yml +31 -31
  7. data/config/checkstyle/checkstyle.xml +128 -128
  8. data/config/checkstyle/default.xml +108 -108
  9. data/gradle/wrapper/gradle-wrapper.properties +5 -5
  10. data/gradlew +172 -172
  11. data/lib/embulk/input/bigquery_extract_files.rb +3 -3
  12. data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java +378 -383
  13. data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java +519 -495
  14. data/src/main/java/org/embulk/input/bigquery_export_gcs/PHASE.java +7 -7
  15. data/src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java +33 -33
  16. data/src/test/java/org/embulk/input/bigquery_export_gcs/TestPluginFunctions.java +56 -56
  17. data/src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java +96 -96
  18. metadata +18 -15
data/gradle/wrapper/gradle-wrapper.properties CHANGED
@@ -1,5 +1,5 @@
1
- distributionBase=GRADLE_USER_HOME
2
- distributionPath=wrapper/dists
3
- zipStoreBase=GRADLE_USER_HOME
4
- zipStorePath=wrapper/dists
5
- distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip
1
+ distributionBase=GRADLE_USER_HOME
2
+ distributionPath=wrapper/dists
3
+ zipStoreBase=GRADLE_USER_HOME
4
+ zipStorePath=wrapper/dists
5
+ distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip
data/gradlew CHANGED
@@ -1,172 +1,172 @@
1
- #!/usr/bin/env sh
2
-
3
- ##############################################################################
4
- ##
5
- ## Gradle start up script for UN*X
6
- ##
7
- ##############################################################################
8
-
9
- # Attempt to set APP_HOME
10
- # Resolve links: $0 may be a link
11
- PRG="$0"
12
- # Need this for relative symlinks.
13
- while [ -h "$PRG" ] ; do
14
- ls=`ls -ld "$PRG"`
15
- link=`expr "$ls" : '.*-> \(.*\)$'`
16
- if expr "$link" : '/.*' > /dev/null; then
17
- PRG="$link"
18
- else
19
- PRG=`dirname "$PRG"`"/$link"
20
- fi
21
- done
22
- SAVED="`pwd`"
23
- cd "`dirname \"$PRG\"`/" >/dev/null
24
- APP_HOME="`pwd -P`"
25
- cd "$SAVED" >/dev/null
26
-
27
- APP_NAME="Gradle"
28
- APP_BASE_NAME=`basename "$0"`
29
-
30
- # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31
- DEFAULT_JVM_OPTS=""
32
-
33
- # Use the maximum available, or set MAX_FD != -1 to use that value.
34
- MAX_FD="maximum"
35
-
36
- warn () {
37
- echo "$*"
38
- }
39
-
40
- die () {
41
- echo
42
- echo "$*"
43
- echo
44
- exit 1
45
- }
46
-
47
- # OS specific support (must be 'true' or 'false').
48
- cygwin=false
49
- msys=false
50
- darwin=false
51
- nonstop=false
52
- case "`uname`" in
53
- CYGWIN* )
54
- cygwin=true
55
- ;;
56
- Darwin* )
57
- darwin=true
58
- ;;
59
- MINGW* )
60
- msys=true
61
- ;;
62
- NONSTOP* )
63
- nonstop=true
64
- ;;
65
- esac
66
-
67
- CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
68
-
69
- # Determine the Java command to use to start the JVM.
70
- if [ -n "$JAVA_HOME" ] ; then
71
- if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
72
- # IBM's JDK on AIX uses strange locations for the executables
73
- JAVACMD="$JAVA_HOME/jre/sh/java"
74
- else
75
- JAVACMD="$JAVA_HOME/bin/java"
76
- fi
77
- if [ ! -x "$JAVACMD" ] ; then
78
- die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
79
-
80
- Please set the JAVA_HOME variable in your environment to match the
81
- location of your Java installation."
82
- fi
83
- else
84
- JAVACMD="java"
85
- which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
86
-
87
- Please set the JAVA_HOME variable in your environment to match the
88
- location of your Java installation."
89
- fi
90
-
91
- # Increase the maximum file descriptors if we can.
92
- if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
93
- MAX_FD_LIMIT=`ulimit -H -n`
94
- if [ $? -eq 0 ] ; then
95
- if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
96
- MAX_FD="$MAX_FD_LIMIT"
97
- fi
98
- ulimit -n $MAX_FD
99
- if [ $? -ne 0 ] ; then
100
- warn "Could not set maximum file descriptor limit: $MAX_FD"
101
- fi
102
- else
103
- warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
104
- fi
105
- fi
106
-
107
- # For Darwin, add options to specify how the application appears in the dock
108
- if $darwin; then
109
- GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
110
- fi
111
-
112
- # For Cygwin, switch paths to Windows format before running java
113
- if $cygwin ; then
114
- APP_HOME=`cygpath --path --mixed "$APP_HOME"`
115
- CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
116
- JAVACMD=`cygpath --unix "$JAVACMD"`
117
-
118
- # We build the pattern for arguments to be converted via cygpath
119
- ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
120
- SEP=""
121
- for dir in $ROOTDIRSRAW ; do
122
- ROOTDIRS="$ROOTDIRS$SEP$dir"
123
- SEP="|"
124
- done
125
- OURCYGPATTERN="(^($ROOTDIRS))"
126
- # Add a user-defined pattern to the cygpath arguments
127
- if [ "$GRADLE_CYGPATTERN" != "" ] ; then
128
- OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
129
- fi
130
- # Now convert the arguments - kludge to limit ourselves to /bin/sh
131
- i=0
132
- for arg in "$@" ; do
133
- CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
134
- CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
135
-
136
- if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
137
- eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
138
- else
139
- eval `echo args$i`="\"$arg\""
140
- fi
141
- i=$((i+1))
142
- done
143
- case $i in
144
- (0) set -- ;;
145
- (1) set -- "$args0" ;;
146
- (2) set -- "$args0" "$args1" ;;
147
- (3) set -- "$args0" "$args1" "$args2" ;;
148
- (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
149
- (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
150
- (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
151
- (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
152
- (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
153
- (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
154
- esac
155
- fi
156
-
157
- # Escape application args
158
- save () {
159
- for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160
- echo " "
161
- }
162
- APP_ARGS=$(save "$@")
163
-
164
- # Collect all arguments for the java command, following the shell quoting and substitution rules
165
- eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166
-
167
- # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168
- if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169
- cd "$(dirname "$0")"
170
- fi
171
-
172
- exec "$JAVACMD" "$@"
1
+ #!/usr/bin/env sh
2
+
3
+ ##############################################################################
4
+ ##
5
+ ## Gradle start up script for UN*X
6
+ ##
7
+ ##############################################################################
8
+
9
+ # Attempt to set APP_HOME
10
+ # Resolve links: $0 may be a link
11
+ PRG="$0"
12
+ # Need this for relative symlinks.
13
+ while [ -h "$PRG" ] ; do
14
+ ls=`ls -ld "$PRG"`
15
+ link=`expr "$ls" : '.*-> \(.*\)$'`
16
+ if expr "$link" : '/.*' > /dev/null; then
17
+ PRG="$link"
18
+ else
19
+ PRG=`dirname "$PRG"`"/$link"
20
+ fi
21
+ done
22
+ SAVED="`pwd`"
23
+ cd "`dirname \"$PRG\"`/" >/dev/null
24
+ APP_HOME="`pwd -P`"
25
+ cd "$SAVED" >/dev/null
26
+
27
+ APP_NAME="Gradle"
28
+ APP_BASE_NAME=`basename "$0"`
29
+
30
+ # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31
+ DEFAULT_JVM_OPTS=""
32
+
33
+ # Use the maximum available, or set MAX_FD != -1 to use that value.
34
+ MAX_FD="maximum"
35
+
36
+ warn () {
37
+ echo "$*"
38
+ }
39
+
40
+ die () {
41
+ echo
42
+ echo "$*"
43
+ echo
44
+ exit 1
45
+ }
46
+
47
+ # OS specific support (must be 'true' or 'false').
48
+ cygwin=false
49
+ msys=false
50
+ darwin=false
51
+ nonstop=false
52
+ case "`uname`" in
53
+ CYGWIN* )
54
+ cygwin=true
55
+ ;;
56
+ Darwin* )
57
+ darwin=true
58
+ ;;
59
+ MINGW* )
60
+ msys=true
61
+ ;;
62
+ NONSTOP* )
63
+ nonstop=true
64
+ ;;
65
+ esac
66
+
67
+ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
68
+
69
+ # Determine the Java command to use to start the JVM.
70
+ if [ -n "$JAVA_HOME" ] ; then
71
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
72
+ # IBM's JDK on AIX uses strange locations for the executables
73
+ JAVACMD="$JAVA_HOME/jre/sh/java"
74
+ else
75
+ JAVACMD="$JAVA_HOME/bin/java"
76
+ fi
77
+ if [ ! -x "$JAVACMD" ] ; then
78
+ die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
79
+
80
+ Please set the JAVA_HOME variable in your environment to match the
81
+ location of your Java installation."
82
+ fi
83
+ else
84
+ JAVACMD="java"
85
+ which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
86
+
87
+ Please set the JAVA_HOME variable in your environment to match the
88
+ location of your Java installation."
89
+ fi
90
+
91
+ # Increase the maximum file descriptors if we can.
92
+ if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
93
+ MAX_FD_LIMIT=`ulimit -H -n`
94
+ if [ $? -eq 0 ] ; then
95
+ if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
96
+ MAX_FD="$MAX_FD_LIMIT"
97
+ fi
98
+ ulimit -n $MAX_FD
99
+ if [ $? -ne 0 ] ; then
100
+ warn "Could not set maximum file descriptor limit: $MAX_FD"
101
+ fi
102
+ else
103
+ warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
104
+ fi
105
+ fi
106
+
107
+ # For Darwin, add options to specify how the application appears in the dock
108
+ if $darwin; then
109
+ GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
110
+ fi
111
+
112
+ # For Cygwin, switch paths to Windows format before running java
113
+ if $cygwin ; then
114
+ APP_HOME=`cygpath --path --mixed "$APP_HOME"`
115
+ CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
116
+ JAVACMD=`cygpath --unix "$JAVACMD"`
117
+
118
+ # We build the pattern for arguments to be converted via cygpath
119
+ ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
120
+ SEP=""
121
+ for dir in $ROOTDIRSRAW ; do
122
+ ROOTDIRS="$ROOTDIRS$SEP$dir"
123
+ SEP="|"
124
+ done
125
+ OURCYGPATTERN="(^($ROOTDIRS))"
126
+ # Add a user-defined pattern to the cygpath arguments
127
+ if [ "$GRADLE_CYGPATTERN" != "" ] ; then
128
+ OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
129
+ fi
130
+ # Now convert the arguments - kludge to limit ourselves to /bin/sh
131
+ i=0
132
+ for arg in "$@" ; do
133
+ CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
134
+ CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
135
+
136
+ if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
137
+ eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
138
+ else
139
+ eval `echo args$i`="\"$arg\""
140
+ fi
141
+ i=$((i+1))
142
+ done
143
+ case $i in
144
+ (0) set -- ;;
145
+ (1) set -- "$args0" ;;
146
+ (2) set -- "$args0" "$args1" ;;
147
+ (3) set -- "$args0" "$args1" "$args2" ;;
148
+ (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
149
+ (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
150
+ (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
151
+ (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
152
+ (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
153
+ (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
154
+ esac
155
+ fi
156
+
157
+ # Escape application args
158
+ save () {
159
+ for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160
+ echo " "
161
+ }
162
+ APP_ARGS=$(save "$@")
163
+
164
+ # Collect all arguments for the java command, following the shell quoting and substitution rules
165
+ eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166
+
167
+ # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168
+ if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169
+ cd "$(dirname "$0")"
170
+ fi
171
+
172
+ exec "$JAVACMD" "$@"
data/lib/embulk/input/bigquery_extract_files.rb CHANGED
@@ -1,3 +1,3 @@
1
- Embulk::JavaPlugin.register_input(
2
- "bigquery_extract_files", "org.embulk.input.bigquery_export_gcs.BigqueryExportGcsFileInputPlugin",
3
- File.expand_path('../../../../classpath', __FILE__))
1
+ Embulk::JavaPlugin.register_input(
2
+ "bigquery_extract_files", "org.embulk.input.bigquery_export_gcs.BigqueryExportGcsFileInputPlugin",
3
+ File.expand_path('../../../../classpath', __FILE__))
data/src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java CHANGED
@@ -1,383 +1,378 @@
1
- package org.embulk.input.bigquery_export_gcs;
2
-
3
- import java.io.File;
4
- import java.io.FileNotFoundException;
5
- import java.io.IOException;
6
- import java.io.InputStream;
7
- import java.nio.file.Path;
8
- import java.util.List;
9
-
10
- import org.codehaus.plexus.util.StringUtils;
11
- import org.embulk.config.Config;
12
- import org.embulk.config.ConfigDefault;
13
- import org.embulk.config.ConfigDiff;
14
- import org.embulk.config.ConfigInject;
15
- import org.embulk.config.ConfigSource;
16
- import org.embulk.config.Task;
17
- import org.embulk.config.TaskReport;
18
- import org.embulk.config.TaskSource;
19
- import org.embulk.exec.ConfigurableGuessInputPlugin;
20
- import org.embulk.exec.GuessExecutor;
21
- import org.embulk.spi.BufferAllocator;
22
- import org.embulk.spi.Exec;
23
- import org.embulk.spi.FileInputPlugin;
24
- import org.embulk.spi.Schema;
25
- import org.embulk.spi.TransactionalFileInput;
26
- import org.embulk.spi.util.InputStreamTransactionalFileInput;
27
- import org.slf4j.Logger;
28
-
29
- import com.google.api.services.bigquery.Bigquery;
30
- import com.google.common.base.Optional;
31
-
32
- import io.airlift.slice.RuntimeIOException;
33
-
34
- /**
35
- *
36
- *
37
- *
38
- * #reference :
39
- *
40
- * # https://github.com/embulk/embulk
41
- * # https://github.com/embulk/embulk-input-s3
42
- * # https://github.com/embulk/embulk-input-gcs
43
- * # https://github.com/embulk/embulk-input-jdbc
44
- * # https://github.com/GoogleCloudPlatform/java-docs-samples/blob/master/storage/json-api/src/main/java/StorageSample.java
45
- *
46
- *
47
- * @author george 2017. 11. 16.
48
- *
49
- */
50
- public class BigqueryExportGcsFileInputPlugin
51
- implements FileInputPlugin, ConfigurableGuessInputPlugin
52
- {
53
- private static final Logger log = Exec.getLogger(BigqueryExportGcsFileInputPlugin.class);
54
-
55
- public interface PluginTask
56
- extends Task
57
- {
58
- @Config("project")
59
- public String getProject();
60
-
61
- @Config("json_keyfile")
62
- public String getJsonKeyfile();
63
-
64
- @Config("dataset")
65
- @ConfigDefault("null")
66
- public Optional<String> getDataset();
67
-
68
- @Config("table")
69
- @ConfigDefault("null")
70
- public Optional<String> getTable();
71
-
72
- @Config("query")
73
- @ConfigDefault("null")
74
- public Optional<String> getQuery();
75
-
76
- @Config("file_format")
77
- @ConfigDefault("\"CSV\"")
78
- public Optional<String> getFileFormat();
79
-
80
- @Config("compression")
81
- @ConfigDefault("\"GZIP\"")
82
- public Optional<String> getCompression();
83
-
84
- @Config("gcs_uri")
85
- public String getGcsUri();
86
-
87
- @Config("temp_dataset")
88
- @ConfigDefault("null")
89
- public Optional<String> getTempDataset();
90
- public void setTempDataset(Optional<String> tempDataset);
91
-
92
- @Config("temp_table")
93
- @ConfigDefault("null")
94
- public Optional<String> getTempTable();
95
- public void setTempTable(Optional<String> tempTable);
96
-
97
- @Config("cache")
98
- @ConfigDefault("true")
99
- public boolean getQueryCache();
100
-
101
- @Config("use_legacy_sql")
102
- @ConfigDefault("false")
103
- public boolean getUseLegacySql();
104
-
105
- @Config("create_disposition")
106
- @ConfigDefault("\"CREATE_IF_NEEDED\"")
107
- public String getCreateDisposition();
108
-
109
- @Config("write_disposition")
110
- @ConfigDefault("\"WRITE_APPEND\"")
111
- public String getWriteDisposition();
112
-
113
- @Config("temp_local_path")
114
- public String getTempLocalPath();
115
-
116
- @Config("temp_schema_file_path")
117
- @ConfigDefault("null")
118
- public Optional<String> getTempSchemaFilePath();
119
-
120
- @Config("temp_schema_file_type")
121
- @ConfigDefault("null")
122
- public Optional<String> getTempSchemaFileType();
123
-
124
- @Config("bigquery_job_wait_second")
125
- @ConfigDefault("600")
126
- public Optional<Integer> getBigqueryJobWaitingSecond();
127
-
128
- @Config("cleanup_gcs_files")
129
- @ConfigDefault("false")
130
- public boolean getCleanupGcsTempFiles();
131
-
132
- @Config("cleanup_temp_table")
133
- @ConfigDefault("true")
134
- public boolean getCleanupTempTable();
135
-
136
- @Config("cleanup_local_temp_files")
137
- @ConfigDefault("true")
138
- public boolean getCleanupLocalTempFiles();
139
-
140
- @Config("cleanup_gcs_before_executing")
141
- @ConfigDefault("true")
142
- public boolean getCleanupGcsBeforeExcuting();
143
-
144
-
145
- @Config("start_phase")
146
- @ConfigDefault("0")
147
- public int getStartPhase();
148
-
149
- public List<String> getFiles();
150
- public void setFiles(List<String> files);
151
-
152
- @ConfigInject
153
- public BufferAllocator getBufferAllocator();
154
-
155
- public String getGcsBucket();
156
- public void setGcsBucket(String bucket);
157
-
158
- public String getGcsBlobNamePrefix();
159
- public void setGcsBlobNamePrefix(String blobName);
160
-
161
- public String getWorkDataset();
162
- public void setWorkDataset(String dataset);
163
-
164
- public String getWorkTable();
165
- public void setWorkTable(String table);
166
-
167
- public String getWorkId();
168
- public void setWorkId(String temp);
169
-
170
- //public Schema getSchemaConfig();
171
- //public void setSchameConfig(SchemaConfig schema);
172
- }
173
-
174
- @Override
175
- public ConfigDiff guess(ConfigSource execConfig, ConfigSource inputConfig) {
176
-
177
- GuessExecutor guessExecutor = Exec.getInjector().getInstance(GuessExecutor.class);
178
- return guessExecutor.guessParserConfig(null, inputConfig, execConfig);
179
- }
180
-
181
- @Override
182
- public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
183
- {
184
- PluginTask task = config.loadConfig(PluginTask.class);
185
-
186
- checkLocalPath(task);
187
-
188
- executeBigqueryApi(task);
189
-
190
- int taskCount = task.getFiles().size();
191
-
192
- return resume(task.dump(), taskCount, control);
193
- }
194
-
195
- public void checkLocalPath(PluginTask task){
196
- File localPath = new File(task.getTempLocalPath());
197
- if(localPath.exists() == false){
198
- log.error("local download path not exists : {}",localPath);
199
- log.info("create local downlaod path : {}", localPath);
200
- boolean ok = localPath.mkdirs();
201
- if(!ok){
202
- throw new RuntimeIOException(new IOException("local path create fail : " + localPath));
203
- }
204
- }
205
- }
206
-
207
- public void executeBigqueryApi(PluginTask task) {
208
-
209
- log.info("[0] Initialize Settings ... ");
210
-
211
- BigqueryExportUtils.parseGcsUri(task);
212
-
213
- if(task.getCleanupGcsBeforeExcuting()){
214
- log.info("clean up before executing. delete all file in : {}",task.getGcsUri());
215
- BigqueryExportUtils.removeGcsFilesBeforeExecuting(task);
216
- }
217
-
218
- PHASE phase = BigqueryExportUtils.initTask(task);
219
- log.info("[0] Settings : {}", BigqueryExportUtils.toPrettyString(task.dump().deepCopy().getObjectNode()) );
220
-
221
- Bigquery bigquery = BigqueryExportUtils.newBigqueryClient(task);
222
-
223
- if(phase == PHASE.QUERY){
224
- log.info("[1] Query to Table");
225
- extractQueryToTable(bigquery, task);
226
-
227
- }
228
- log.info("[2] Table to GCS");
229
- Schema schema = extractTableToGcs(bigquery, task);
230
- log.info("Schema : {}",schema.toString());
231
-
232
- log.info("[3] Write Schema ");
233
- writeSchemaFileIfSpecified(schema, task);
234
-
235
- log.info("[4] read file list in gcs ");
236
- List<String> files = listFilesOfGcs(task);
237
-
238
- task.setFiles(files);
239
-
240
- }
241
-
242
- public void writeSchemaFileIfSpecified(Schema schema, PluginTask task) {
243
- if(task.getTempSchemaFilePath().isPresent()) {
244
- log.info("generate temp {} schema file to ... {}", task.getTempSchemaFileType().or(""), task.getTempSchemaFilePath().orNull());
245
- BigqueryExportUtils.writeSchemaFile(schema, task.getTempSchemaFileType().orNull(), new File(task.getTempSchemaFilePath().get()));
246
- }
247
- }
248
-
249
- public void extractQueryToTable(Bigquery bigquery, PluginTask task){
250
- try {
251
- BigqueryExportUtils.executeQueryToDestinationWorkTable(bigquery, task);
252
- } catch (IOException e) {
253
- log.error("bigquery io error",e);
254
- throw new RuntimeIOException(e);
255
- } catch (InterruptedException e) {
256
- log.error("bigquery job error",e);
257
- throw new RuntimeException(e);
258
- }
259
- }
260
-
261
- public Schema extractTableToGcs(Bigquery bigquery, PluginTask task){
262
- try {
263
- // extract table and get schema
264
- Schema schema = BigqueryExportUtils.extractWorkTable(bigquery, task);
265
- return schema;
266
- } catch (IOException e) {
267
- log.error("bigquery io error",e);
268
- throw new RuntimeIOException(e);
269
- } catch (InterruptedException e) {
270
- log.error("bigquery job error",e);
271
- throw new RuntimeException(e);
272
- }
273
- }
274
-
275
- // usually, you have an method to create list of files
276
- List<String> listFilesOfGcs(PluginTask task)
277
- {
278
- log.info("get file list in to gcs of ... {}.{} -> gs://{}/{}", task.getWorkDataset(), task.getWorkTable(),task.getGcsBucket(),task.getGcsBlobNamePrefix());
279
-
280
- try {
281
- return BigqueryExportUtils.getFileListFromGcs(task);
282
- } catch (IOException e) {
283
- log.error("GCS api call error");
284
- throw new RuntimeIOException(e);
285
- }
286
-
287
- }
288
-
289
-
290
- @Override
291
- public ConfigDiff resume(TaskSource taskSource,
292
- int taskCount,
293
- FileInputPlugin.Control control)
294
- {
295
- control.run(taskSource, taskCount);
296
-
297
- ConfigDiff configDiff = Exec.newConfigDiff();
298
- //configDiff.has(attrName)
299
-
300
-
301
- // usually, yo uset last_path
302
- //if (task.getFiles().isEmpty()) {
303
- // if (task.getLastPath().isPresent()) {
304
- // configDiff.set("last_path", task.getLastPath().get());
305
- // }
306
- //} else {
307
- // List<String> files = new ArrayList<String>(task.getFiles());
308
- // Collections.sort(files);
309
- // configDiff.set("last_path", files.get(files.size() - 1));
310
- //}
311
-
312
- return configDiff;
313
- }
314
-
315
- @Override
316
- public void cleanup(TaskSource taskSource,
317
- int taskCount,
318
- List<TaskReport> successTaskReports)
319
- {
320
- final PluginTask task = taskSource.loadTask(PluginTask.class);
321
-
322
- // remove query temp table when exists
323
- if(task.getCleanupTempTable() &&
324
- task.getTempTable().isPresent() &&
325
- task.getQuery().isPresent() &&
326
- task.getTempDataset().isPresent()){
327
- BigqueryExportUtils.removeTempTable(task);
328
- }
329
-
330
- for(int i=0; i < successTaskReports.size(); i++){
331
- TaskReport report = successTaskReports.get(i);
332
- if( report.isEmpty() ){
333
- String file = task.getFiles().get(i);
334
-
335
- if(task.getCleanupLocalTempFiles()) {
336
- Path p = BigqueryExportUtils.getFullPath(task,file);
337
- log.info("delete temp file...{}",p);
338
- p.toFile().delete();
339
- }
340
-
341
- if(task.getCleanupGcsTempFiles()){
342
- BigqueryExportUtils.removeTempGcsFiles(task, file);
343
- }
344
-
345
- //
346
- }else{
347
- log.error("datasource not empty : {}", report);
348
- }
349
- }
350
-
351
- }
352
-
353
-
354
- @Override
355
- public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
356
- {
357
- final PluginTask task = taskSource.loadTask(PluginTask.class);
358
-
359
- // Write your code here :)
360
- //throw new UnsupportedOperationException("BigquerycsvFileInputPlugin.open method is not implemented yet");
361
-
362
- // if you expect InputStream, you can use this code:
363
-
364
- InputStream input = BigqueryExportUtils.openInputStream(task, task.getFiles().get(taskIndex));
365
-
366
- return new InputStreamTransactionalFileInput(task.getBufferAllocator(), input) {
367
- @Override
368
- public void abort()
369
- { }
370
-
371
- @Override
372
- public TaskReport commit()
373
- {
374
- return Exec.newTaskReport();
375
- }
376
- };
377
- }
378
-
379
- //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
380
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
381
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
382
-
383
- }
1
+ package org.embulk.input.bigquery_export_gcs;
2
+
3
+ import java.io.File;
4
+ import java.io.FileNotFoundException;
5
+ import java.io.IOException;
6
+ import java.io.InputStream;
7
+ import java.nio.file.Path;
8
+ import java.util.List;
9
+
10
+ import org.codehaus.plexus.util.StringUtils;
11
+ import org.embulk.config.Config;
12
+ import org.embulk.config.ConfigDefault;
13
+ import org.embulk.config.ConfigDiff;
14
+ import org.embulk.config.ConfigInject;
15
+ import org.embulk.config.ConfigSource;
16
+ import org.embulk.config.Task;
17
+ import org.embulk.config.TaskReport;
18
+ import org.embulk.config.TaskSource;
19
+ import org.embulk.exec.ConfigurableGuessInputPlugin;
20
+ import org.embulk.exec.GuessExecutor;
21
+ import org.embulk.spi.BufferAllocator;
22
+ import org.embulk.spi.Exec;
23
+ import org.embulk.spi.FileInputPlugin;
24
+ import org.embulk.spi.Schema;
25
+ import org.embulk.spi.TransactionalFileInput;
26
+ import org.embulk.spi.util.InputStreamTransactionalFileInput;
27
+ import org.slf4j.Logger;
28
+
29
+ import com.google.api.services.bigquery.Bigquery;
30
+ import com.google.common.base.Optional;
31
+
32
+ import io.airlift.slice.RuntimeIOException;
33
+
34
+ /**
35
+ *
36
+ *
37
+ *
38
+ * #reference :
39
+ *
40
+ * # https://github.com/embulk/embulk
41
+ * # https://github.com/embulk/embulk-input-s3
42
+ * # https://github.com/embulk/embulk-input-gcs
43
+ * # https://github.com/embulk/embulk-input-jdbc
44
+ * # https://github.com/GoogleCloudPlatform/java-docs-samples/blob/master/storage/json-api/src/main/java/StorageSample.java
45
+ *
46
+ *
47
+ * @author george 2017. 11. 16.
48
+ *
49
+ */
50
+ public class BigqueryExportGcsFileInputPlugin implements FileInputPlugin
51
+ {
52
+ private static final Logger log = Exec.getLogger(BigqueryExportGcsFileInputPlugin.class);
53
+
54
+ public interface PluginTask
55
+ extends Task
56
+ {
57
+ @Config("project")
58
+ public String getProject();
59
+
60
+ @Config("json_keyfile")
61
+ public String getJsonKeyfile();
62
+
63
+ @Config("dataset")
64
+ @ConfigDefault("null")
65
+ public Optional<String> getDataset();
66
+
67
+ @Config("table")
68
+ @ConfigDefault("null")
69
+ public Optional<String> getTable();
70
+
71
+ @Config("query")
72
+ @ConfigDefault("null")
73
+ public Optional<String> getQuery();
74
+
75
+ @Config("file_format")
76
+ @ConfigDefault("\"CSV\"")
77
+ public Optional<String> getFileFormat();
78
+
79
+ @Config("compression")
80
+ @ConfigDefault("\"GZIP\"")
81
+ public Optional<String> getCompression();
82
+
83
+ @Config("gcs_uri")
84
+ public String getGcsUri();
85
+
86
+ @Config("temp_dataset")
87
+ @ConfigDefault("null")
88
+ public Optional<String> getTempDataset();
89
+ public void setTempDataset(Optional<String> tempDataset);
90
+
91
+ @Config("temp_table")
92
+ @ConfigDefault("null")
93
+ public Optional<String> getTempTable();
94
+ public void setTempTable(Optional<String> tempTable);
95
+
96
+ @Config("cache")
97
+ @ConfigDefault("true")
98
+ public boolean getQueryCache();
99
+
100
+ @Config("use_legacy_sql")
101
+ @ConfigDefault("false")
102
+ public boolean getUseLegacySql();
103
+
104
+ @Config("create_disposition")
105
+ @ConfigDefault("\"CREATE_IF_NEEDED\"")
106
+ public String getCreateDisposition();
107
+
108
+ @Config("write_disposition")
109
+ @ConfigDefault("\"WRITE_APPEND\"")
110
+ public String getWriteDisposition();
111
+
112
+ @Config("temp_local_path")
113
+ public String getTempLocalPath();
114
+
115
+ @Config("temp_schema_file_path")
116
+ @ConfigDefault("null")
117
+ public Optional<String> getTempSchemaFilePath();
118
+
119
+ @Config("temp_schema_file_type")
120
+ @ConfigDefault("null")
121
+ public Optional<String> getTempSchemaFileType();
122
+
123
+ @Config("bigquery_job_wait_second")
124
+ @ConfigDefault("600")
125
+ public Optional<Integer> getBigqueryJobWaitingSecond();
126
+
127
+ @Config("cleanup_gcs_files")
128
+ @ConfigDefault("false")
129
+ public boolean getCleanupGcsTempFiles();
130
+
131
+ @Config("cleanup_temp_table")
132
+ @ConfigDefault("true")
133
+ public boolean getCleanupTempTable();
134
+
135
+ @Config("cleanup_local_temp_files")
136
+ @ConfigDefault("true")
137
+ public boolean getCleanupLocalTempFiles();
138
+
139
+ @Config("cleanup_gcs_before_executing")
140
+ @ConfigDefault("true")
141
+ public boolean getCleanupGcsBeforeExcuting();
142
+
143
+ @Config("guess_schema")
144
+ @ConfigDefault("true")
145
+ public boolean getGuessSchema();
146
+
147
+ @Config("start_phase")
148
+ @ConfigDefault("0")
149
+ public int getStartPhase();
150
+
151
+ public List<String> getFiles();
152
+ public void setFiles(List<String> files);
153
+
154
+ @ConfigInject
155
+ public BufferAllocator getBufferAllocator();
156
+
157
+ public String getGcsBucket();
158
+ public void setGcsBucket(String bucket);
159
+
160
+ public String getGcsBlobNamePrefix();
161
+ public void setGcsBlobNamePrefix(String blobName);
162
+
163
+ public String getWorkDataset();
164
+ public void setWorkDataset(String dataset);
165
+
166
+ public String getWorkTable();
167
+ public void setWorkTable(String table);
168
+
169
+ public String getWorkId();
170
+ public void setWorkId(String temp);
171
+
172
+ //public Schema getSchemaConfig();
173
+ //public void setSchameConfig(SchemaConfig schema);
174
+ }
175
+
176
+ @Override
177
+ public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
178
+ {
179
+ PluginTask task = config.loadConfig(PluginTask.class);
180
+
181
+ checkLocalPath(task);
182
+
183
+ executeBigqueryApi(task);
184
+
185
+ int taskCount = task.getFiles().size();
186
+
187
+ return resume(task.dump(), taskCount, control);
188
+ }
189
+
190
+ public void checkLocalPath(PluginTask task){
191
+ File localPath = new File(task.getTempLocalPath());
192
+ if(localPath.exists() == false){
193
+ log.error("local download path not exists : {}",localPath);
194
+ log.info("create local downlaod path : {}", localPath);
195
+ boolean ok = localPath.mkdirs();
196
+ if(!ok){
197
+ throw new RuntimeIOException(new IOException("local path create fail : " + localPath));
198
+ }
199
+ }
200
+ }
201
+
202
+ public void executeBigqueryApi(PluginTask task) {
203
+
204
+ log.info("[0] Initialize Settings ... ");
205
+
206
+ BigqueryExportUtils.parseGcsUri(task);
207
+
208
+ if(task.getCleanupGcsBeforeExcuting()){
209
+ log.info("clean up before executing. delete all file in : {}",task.getGcsUri());
210
+ BigqueryExportUtils.removeGcsFilesBeforeExecuting(task);
211
+ }
212
+
213
+ PHASE phase = BigqueryExportUtils.initTask(task);
214
+ log.info("[0] Settings : {}", BigqueryExportUtils.toPrettyString(task.dump().deepCopy().getObjectNode()) );
215
+
216
+ Bigquery bigquery = BigqueryExportUtils.newBigqueryClient(task);
217
+
218
+ if(phase == PHASE.QUERY){
219
+ log.info("[1] Query to Table");
220
+ extractQueryToTable(bigquery, task);
221
+
222
+ }
223
+ log.info("[2] Table to GCS");
224
+ Schema schema = extractTableToGcs(bigquery, task);
225
+ log.info("Schema : {}",schema.toString());
226
+
227
+ log.info("[3] Write Schema ");
228
+ writeSchemaFileIfSpecified(schema, task);
229
+
230
+ log.info("[4] read file list in gcs ");
231
+ List<String> files = listFilesOfGcs(task);
232
+
233
+ task.setFiles(files);
234
+
235
+ }
236
+
237
+ public void writeSchemaFileIfSpecified(Schema schema, PluginTask task) {
238
+ if(task.getTempSchemaFilePath().isPresent()) {
239
+ log.info("generate temp {} schema file to ... {}", task.getTempSchemaFileType().or(""), task.getTempSchemaFilePath().orNull());
240
+ BigqueryExportUtils.writeSchemaFile(schema, task.getTempSchemaFileType().orNull(), new File(task.getTempSchemaFilePath().get()));
241
+ }
242
+ }
243
+
244
+ public void extractQueryToTable(Bigquery bigquery, PluginTask task){
245
+ try {
246
+ BigqueryExportUtils.executeQueryToDestinationWorkTable(bigquery, task);
247
+ } catch (IOException e) {
248
+ log.error("bigquery io error",e);
249
+ throw new RuntimeIOException(e);
250
+ } catch (InterruptedException e) {
251
+ log.error("bigquery job error",e);
252
+ throw new RuntimeException(e);
253
+ }
254
+ }
255
+
256
+ public Schema extractTableToGcs(Bigquery bigquery, PluginTask task){
257
+ try {
258
+ // extract table and get schema
259
+ Schema schema = BigqueryExportUtils.extractWorkTable(bigquery, task);
260
+ return schema;
261
+ } catch (IOException e) {
262
+ log.error("bigquery io error",e);
263
+ throw new RuntimeIOException(e);
264
+ } catch (InterruptedException e) {
265
+ log.error("bigquery job error",e);
266
+ throw new RuntimeException(e);
267
+ }
268
+ }
269
+
270
+ // usually, you have an method to create list of files
271
+ List<String> listFilesOfGcs(PluginTask task)
272
+ {
273
+ log.info("get file list in to gcs of ... {}.{} -> gs://{}/{}", task.getWorkDataset(), task.getWorkTable(),task.getGcsBucket(),task.getGcsBlobNamePrefix());
274
+
275
+ try {
276
+ return BigqueryExportUtils.getFileListFromGcs(task);
277
+ } catch (IOException e) {
278
+ log.error("GCS api call error");
279
+ throw new RuntimeIOException(e);
280
+ }
281
+
282
+ }
283
+
284
+
285
+ @Override
286
+ public ConfigDiff resume(TaskSource taskSource,
287
+ int taskCount,
288
+ FileInputPlugin.Control control)
289
+ {
290
+ control.run(taskSource, taskCount);
291
+
292
+ ConfigDiff configDiff = Exec.newConfigDiff();
293
+ //configDiff.has(attrName)
294
+
295
+
296
+ // usually, yo uset last_path
297
+ //if (task.getFiles().isEmpty()) {
298
+ // if (task.getLastPath().isPresent()) {
299
+ // configDiff.set("last_path", task.getLastPath().get());
300
+ // }
301
+ //} else {
302
+ // List<String> files = new ArrayList<String>(task.getFiles());
303
+ // Collections.sort(files);
304
+ // configDiff.set("last_path", files.get(files.size() - 1));
305
+ //}
306
+
307
+ return configDiff;
308
+ }
309
+
310
+ @Override
311
+ public void cleanup(TaskSource taskSource,
312
+ int taskCount,
313
+ List<TaskReport> successTaskReports)
314
+ {
315
+ final PluginTask task = taskSource.loadTask(PluginTask.class);
316
+
317
+ // remove query temp table when exists
318
+ if(task.getCleanupTempTable() &&
319
+ task.getTempTable().isPresent() &&
320
+ task.getQuery().isPresent() &&
321
+ task.getTempDataset().isPresent()){
322
+ BigqueryExportUtils.removeTempTable(task);
323
+ }
324
+
325
+ for(int i=0; i < successTaskReports.size(); i++){
326
+ TaskReport report = successTaskReports.get(i);
327
+ if( report.isEmpty() ){
328
+ String file = task.getFiles().get(i);
329
+
330
+ if(task.getCleanupLocalTempFiles()) {
331
+ Path p = BigqueryExportUtils.getFullPath(task,file);
332
+ log.info("delete temp file...{}",p);
333
+ p.toFile().delete();
334
+ }
335
+
336
+ if(task.getCleanupGcsTempFiles()){
337
+ BigqueryExportUtils.removeTempGcsFiles(task, file);
338
+ }
339
+
340
+ //
341
+ }else{
342
+ log.error("datasource not empty : {}", report);
343
+ }
344
+ }
345
+
346
+ }
347
+
348
+
349
+ @Override
350
+ public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
351
+ {
352
+ final PluginTask task = taskSource.loadTask(PluginTask.class);
353
+
354
+ // Write your code here :)
355
+ //throw new UnsupportedOperationException("BigquerycsvFileInputPlugin.open method is not implemented yet");
356
+
357
+ // if you expect InputStream, you can use this code:
358
+
359
+ InputStream input = BigqueryExportUtils.openInputStream(task, task.getFiles().get(taskIndex));
360
+
361
+ return new InputStreamTransactionalFileInput(task.getBufferAllocator(), input) {
362
+ @Override
363
+ public void abort()
364
+ { }
365
+
366
+ @Override
367
+ public TaskReport commit()
368
+ {
369
+ return Exec.newTaskReport();
370
+ }
371
+ };
372
+ }
373
+
374
+ //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
375
+ ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
376
+ ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
377
+
378
+ }