embulk-output-bigquery 0.2.3 → 0.3.0.pre1

data/gradlew.bat

@@ -1,90 +0,0 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windowz variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega

data/settings.gradle

@@ -1,2 +0,0 @@
-rootProject.name = 'embulk-output-bigquery'
-

data/src/main/java/org/embulk/output/BigqueryAuthentication.java

@@ -1,117 +0,0 @@
-package org.embulk.output;
-
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.compute.ComputeCredential;
-import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.api.client.http.HttpRequestInitializer;
-import com.google.api.client.http.HttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.bigquery.Bigquery;
-import com.google.api.services.bigquery.BigqueryScopes;
-import com.google.api.services.bigquery.model.ProjectList;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
-import org.embulk.spi.Exec;
-import org.slf4j.Logger;
-
-import java.io.File;
-import java.io.FileInputStream;
-
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-import java.util.Collections;
-
-public class BigqueryAuthentication
-{
-    private final Logger log = Exec.getLogger(BigqueryAuthentication.class);
-    private final Optional<String> serviceAccountEmail;
-    private final Optional<String> p12KeyFilePath;
-    private final Optional<String> jsonKeyFilePath;
-    private final String applicationName;
-    private final HttpTransport httpTransport;
-    private final JsonFactory jsonFactory;
-    private final HttpRequestInitializer credentials;
-
-    public BigqueryAuthentication(String authMethod, Optional<String> serviceAccountEmail,
-            Optional<String> p12KeyFilePath, Optional<String> jsonKeyFilePath, String applicationName)
-            throws IOException, GeneralSecurityException
-    {
-        this.serviceAccountEmail = serviceAccountEmail;
-        this.p12KeyFilePath = p12KeyFilePath;
-        this.jsonKeyFilePath = jsonKeyFilePath;
-        this.applicationName = applicationName;
-
-        this.httpTransport = GoogleNetHttpTransport.newTrustedTransport();
-        this.jsonFactory = new JacksonFactory();
-
-        if (authMethod.toLowerCase().equals("compute_engine")) {
-            this.credentials = getComputeCredential();
-        }
-        else if (authMethod.toLowerCase().equals("json_key")) {
-            this.credentials = getServiceAccountCredentialFromJsonFile();
-        }
-        else {
-            this.credentials = getServiceAccountCredential();
-        }
-    }
-
-    /**
-     * @see https://developers.google.com/accounts/docs/OAuth2ServiceAccount#authorizingrequests
-     */
-    private GoogleCredential getServiceAccountCredential() throws IOException, GeneralSecurityException
-    {
-        // @see https://cloud.google.com/compute/docs/api/how-tos/authorization
-        // @see https://developers.google.com/resources/api-libraries/documentation/storage/v1/java/latest/com/google/api/services/storage/STORAGE_SCOPE.html
-        // @see https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/BigqueryScopes.html
-        return new GoogleCredential.Builder()
-                .setTransport(httpTransport)
-                .setJsonFactory(jsonFactory)
-                .setServiceAccountId(serviceAccountEmail.orNull())
-                .setServiceAccountScopes(
-                        ImmutableList.of(
-                                BigqueryScopes.BIGQUERY
-                        )
-                )
-                .setServiceAccountPrivateKeyFromP12File(new File(p12KeyFilePath.orNull()))
-                .build();
-    }
-
-    private GoogleCredential getServiceAccountCredentialFromJsonFile() throws IOException
-    {
-        FileInputStream stream = new FileInputStream(jsonKeyFilePath.orNull());
-
-        return GoogleCredential.fromStream(stream, httpTransport, jsonFactory)
-                .createScoped(Collections.singleton(BigqueryScopes.BIGQUERY));
-    }
-
-    /**
-     * @see http://developers.guge.io/accounts/docs/OAuth2ServiceAccount#creatinganaccount
-     * @see https://developers.google.com/accounts/docs/OAuth2
-     */
-    private ComputeCredential getComputeCredential() throws IOException
-    {
-        ComputeCredential credential = new ComputeCredential.Builder(httpTransport, jsonFactory)
-                .build();
-        credential.refreshToken();
-
-        //log.debug("access_token:" + credential.getAccessToken());
-        log.debug("access_token expired:" + credential.getExpiresInSeconds());
-
-        return credential;
-    }
-
-    public Bigquery getBigqueryClient() throws GoogleJsonResponseException, IOException
-    {
-        Bigquery client = new Bigquery.Builder(httpTransport, jsonFactory, credentials)
-                .setApplicationName(applicationName)
-                .build();
-
-        // For throw IOException when authentication is fail.
-        long maxResults = 1;
-        ProjectList projectList = client.projects().list().setMaxResults(maxResults).execute();
-
-        return client;
-    }
-}

data/src/main/java/org/embulk/output/BigqueryOutputPlugin.java

@@ -1,508 +0,0 @@
-package org.embulk.output;
-
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigException;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileOutputPlugin;
-import org.embulk.spi.TransactionalFileOutput;
-import org.embulk.spi.unit.LocalFile;
-import org.jruby.embed.ScriptingContainer;
-import org.slf4j.Logger;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.security.GeneralSecurityException;
-import java.security.NoSuchAlgorithmException;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-
-public class BigqueryOutputPlugin
-        implements FileOutputPlugin
-{
-    public interface PluginTask
-            extends Task
-    {
-        @Config("auth_method")
-        @ConfigDefault("\"private_key\"")
-        AuthMethod getAuthMethod();
-
-        @Config("service_account_email")
-        @ConfigDefault("null")
-        Optional<String> getServiceAccountEmail();
-
-        // kept for backward compatibility
-        @Config("p12_keyfile_path")
-        @ConfigDefault("null")
-        Optional<String> getP12KeyfilePath();
-
-        @Config("p12_keyfile")
-        @ConfigDefault("null")
-        Optional<LocalFile> getP12Keyfile();
-        void setP12Keyfile(Optional<LocalFile> p12Keyfile);
-
-        @Config("json_keyfile")
-        @ConfigDefault("null")
-        Optional<LocalFile> getJsonKeyfile();
-
-        @Config("application_name")
-        @ConfigDefault("\"Embulk BigQuery plugin\"")
-        String getApplicationName();
-
-        @Config("path_prefix")
-        String getPathPrefix();
-
-        @Config("sequence_format")
-        @ConfigDefault("\".%03d.%02d\"")
-        String getSequenceFormat();
-
-        @Config("file_ext")
-        String getFileNameExtension();
-
-        @Config("source_format")
-        @ConfigDefault("\"CSV\"")
-        SourceFormat getSourceFormat();
-
-        @Config("field_delimiter")
-        @ConfigDefault("\",\"")
-        char getFieldDelimiter();
-
-        @Config("max_bad_records")
-        @ConfigDefault("0")
-        int getMaxBadrecords();
-
-        @Config("encoding")
-        @ConfigDefault("\"UTF-8\"")
-        Charset getEncoding();
-
-        @Config("delete_from_local_when_job_end")
-        @ConfigDefault("false")
-        boolean getDeleteFromLocalWhenJobEnd();
-
-        @Config("project")
-        String getProject();
-
-        @Config("dataset")
-        String getDataset();
-
-        @Config("table")
-        String getTable();
-
-        @Config("auto_create_table")
-        @ConfigDefault("false")
-        boolean getAutoCreateTable();
-
-        // kept for backward compatibility
-        @Config("schema_path")
-        @ConfigDefault("null")
-        Optional<String> getSchemaPath();
-
-        @Config("schema_file")
-        @ConfigDefault("null")
-        Optional<LocalFile> getSchemaFile();
-        void setSchemaFile(Optional<LocalFile> schemaFile);
-
-        @Config("template_table")
-        @ConfigDefault("null")
-        Optional<String> getTemplateTable();
-
-        @Config("prevent_duplicate_insert")
-        @ConfigDefault("false")
-        boolean getPreventDuplicateInsert();
-
-        @Config("job_status_max_polling_time")
-        @ConfigDefault("3600")
-        int getJobStatusMaxPollingTime();
-
-        @Config("job_status_polling_interval")
-        @ConfigDefault("10")
-        int getJobStatusPollingInterval();
-
-        @Config("is_skip_job_result_check")
-        @ConfigDefault("false")
-        boolean getIsSkipJobResultCheck();
-
-        @Config("ignore_unknown_values")
-        @ConfigDefault("false")
-        boolean getIgnoreUnknownValues();
-
-        @Config("allow_quoted_newlines")
-        @ConfigDefault("false")
-        boolean getAllowQuotedNewlines();
-
-        @Config("mode")
-        @ConfigDefault("\"append\"")
-        Mode getMode();
-    }
-
-    private final Logger log = Exec.getLogger(BigqueryOutputPlugin.class);
-    private static final String temporaryTableSuffix = Long.toString(System.currentTimeMillis());
-    private static BigqueryWriter bigQueryWriter;
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config, int taskCount,
-                                  FileOutputPlugin.Control control)
-    {
-        final PluginTask task = config.loadConfig(PluginTask.class);
-
-        if (task.getP12KeyfilePath().isPresent()) {
-            if (task.getP12Keyfile().isPresent()) {
-                throw new ConfigException("Setting both p12_keyfile_path and p12_keyfile is invalid");
-            }
-            try {
-                task.setP12Keyfile(Optional.of(LocalFile.of(task.getP12KeyfilePath().get())));
-            }
-            catch (IOException ex) {
-                throw Throwables.propagate(ex);
-            }
-        }
-
-        if (task.getSchemaPath().isPresent()) {
-            if (task.getSchemaFile().isPresent()) {
-                throw new ConfigException("Setting both p12_keyfile_path and p12_keyfile is invalid");
-            }
-            try {
-                task.setSchemaFile(Optional.of(LocalFile.of(task.getSchemaPath().get())));
-            }
-            catch (IOException ex) {
-                throw Throwables.propagate(ex);
-            }
-        }
-
-        if (task.getAuthMethod().getString().equals("json_key")) {
-            if (!task.getJsonKeyfile().isPresent()) {
-                throw new ConfigException("If auth_method is json_key, you have to set json_keyfile");
-            }
-        }
-        else if (task.getAuthMethod().getString().equals("private_key")) {
-            if (!task.getP12Keyfile().isPresent() || !task.getServiceAccountEmail().isPresent()) {
-                throw new ConfigException("If auth_method is private_key, you have to set both service_account_email and p12_keyfile");
-            }
-        }
-
-        if (task.getMode().isReplaceMode()) {
-            if (task.getIsSkipJobResultCheck()) {
-                throw new ConfigException("If mode is replace or replace_backup, is_skip_job_result_check must be false");
-            }
-        }
-
-        if (task.getMode().isDeleteInAdvance()) {
-            if (!task.getAutoCreateTable()) {
-                throw new ConfigException("If mode is delete_in_advance, auto_create_table must be true");
-            }
-        }
-
-        try {
-            bigQueryWriter = new BigqueryWriter.Builder(
-                    task.getAuthMethod().getString(),
-                    task.getServiceAccountEmail(),
-                    task.getP12Keyfile().transform(localFileToPathString()),
-                    task.getJsonKeyfile().transform(localFileToPathString()),
-                    task.getApplicationName())
-                    .setProject(task.getProject())
-                    .setDataset(task.getDataset())
-                    .setTable(task.getTable())
-                    .setAutoCreateTable(task.getAutoCreateTable())
-                    .setSchemaPath(task.getSchemaFile().transform(localFileToPathString()))
-                    .setTemplateTable(task.getTemplateTable())
-                    .setSourceFormat(task.getSourceFormat().getString())
-                    .setFieldDelimiter(String.valueOf(task.getFieldDelimiter()))
-                    .setMaxBadRecords(task.getMaxBadrecords())
-                    .setEncoding(String.valueOf(task.getEncoding()))
-                    .setPreventDuplicateInsert(task.getPreventDuplicateInsert())
-                    .setJobStatusMaxPollingTime(task.getJobStatusMaxPollingTime())
-                    .setJobStatusPollingInterval(task.getJobStatusPollingInterval())
-                    .setIsSkipJobResultCheck(task.getIsSkipJobResultCheck())
-                    .setIgnoreUnknownValues(task.getIgnoreUnknownValues())
-                    .setAllowQuotedNewlines(task.getAllowQuotedNewlines())
-                    .build();
-        }
-        catch (IOException | GeneralSecurityException ex) {
-            throw new ConfigException(ex);
-        }
-        // non-retryable (non-idempotent) output:
-        return resume(task.dump(), taskCount, control);
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-                             int taskCount,
-                             FileOutputPlugin.Control control)
-    {
-        Mode mode = taskSource.get(Mode.class, "Mode");
-        String project = taskSource.get(String.class, "Project");
-        String dataset = taskSource.get(String.class, "Dataset");
-        String tableName = taskSource.get(String.class, "Table");
-
-        if (mode == Mode.delete_in_advance) {
-            try {
-                bigQueryWriter.deleteTable(project, dataset, generateTableName(tableName));
-            }
-            catch (IOException ex) {
-                log.warn(ex.getMessage());
-            }
-        }
-
-        control.run(taskSource);
-
-        if (mode.isReplaceMode()) {
-            try {
-                if (mode == Mode.replace_backup && bigQueryWriter.isExistTable(project, dataset, generateTableName(tableName))) {
-                    bigQueryWriter.replaceTable(project, dataset, generateTableName(tableName) + "_old", generateTableName(tableName));
-                }
-                bigQueryWriter.replaceTable(project, dataset, generateTableName(tableName), generateTemporaryTableName(tableName));
-            }
-            catch (TimeoutException | BigqueryWriter.JobFailedException | IOException ex) {
-                log.error(ex.getMessage());
-                throw Throwables.propagate(ex);
-            }
-            finally {
-                try {
-                    bigQueryWriter.deleteTable(project, dataset, generateTemporaryTableName(tableName));
-                }
-                catch (IOException ex) {
-                    log.warn(ex.getMessage());
-                }
-            }
-        }
-
-        return Exec.newConfigDiff();
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-                        int taskCount,
-                        List<TaskReport> successTaskReports)
-    {
-    }
-
-    private Function<LocalFile, String> localFileToPathString()
-    {
-        return new Function<LocalFile, String>()
-        {
-            public String apply(LocalFile file)
-            {
-                return file.getPath().toString();
-            }
-        };
-    }
-
-    @Override
-    public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
-    {
-        final PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        final String pathPrefix = task.getPathPrefix();
-        final String sequenceFormat = task.getSequenceFormat();
-        final String pathSuffix = task.getFileNameExtension();
-
-        return new TransactionalFileOutput() {
-            private final String project = task.getProject();
-            private final String dataset = task.getDataset();
-            private final String table = task.getMode().isReplaceMode() ?
-                    generateTemporaryTableName(task.getTable()) : generateTableName(task.getTable());
-            private final boolean deleteFromLocalWhenJobEnd = task.getDeleteFromLocalWhenJobEnd();
-
-            private int fileIndex = 0;
-            private BufferedOutputStream output = null;
-            private File file;
-            private String filePath;
-
-            public void nextFile()
-            {
-                closeFile();
-
-                try {
-                    String suffix = pathSuffix;
-                    if (!suffix.startsWith(".")) {
-                        suffix = "." + suffix;
-                    }
-                    filePath = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + suffix;
-                    file = new File(filePath);
-
-                    String parentPath = file.getParent();
-                    File dir = new File(parentPath);
-                    if (!dir.exists()) {
-                        dir.mkdir();
-                    }
-                    log.info(String.format("Writing file [%s]", filePath));
-                    output = new BufferedOutputStream(new FileOutputStream(filePath));
-                }
-                catch (FileNotFoundException ex) {
-                    throw Throwables.propagate(ex);
-                }
-                fileIndex++;
-            }
-
-            private void closeFile()
-            {
-                if (output != null) {
-                    try {
-                        output.close();
-                    }
-                    catch (IOException ex) {
-                        throw Throwables.propagate(ex);
-                    }
-                }
-            }
-
-            public void add(Buffer buffer)
-            {
-                try {
-                    output.write(buffer.array(), buffer.offset(), buffer.limit());
-                }
-                catch (IOException ex) {
-                    throw Throwables.propagate(ex);
-                }
-                finally {
-                    buffer.release();
-                }
-            }
-
-            public void finish()
-            {
-                closeFile();
-                if (filePath != null) {
-                    try {
-                        bigQueryWriter.executeLoad(project, dataset, table, filePath);
-
-                        if (deleteFromLocalWhenJobEnd) {
-                            log.info(String.format("Delete local file [%s]", filePath));
-                            file.delete();
-                        }
-                    }
-                    catch (NoSuchAlgorithmException | TimeoutException | BigqueryWriter.JobFailedException | IOException ex) {
-                        log.error(ex.getMessage());
-                        throw Throwables.propagate(ex);
-                    }
-                }
-            }
-
-            public void close()
-            {
-                closeFile();
-            }
-
-            public void abort()
-            {
-            }
-
-            public TaskReport commit()
-            {
-                TaskReport report = Exec.newTaskReport();
-                return report;
-            }
-        };
-    }
-
-    // Parse like "table_%Y_%m"(include pattern or not) format using Java is difficult. So use jRuby.
-    public String generateTableName(String tableName)
-    {
-        ScriptingContainer jruby = new ScriptingContainer();
-        Object result = jruby.runScriptlet("Time.now.strftime('" + tableName + "')");
-
-        return result.toString();
-    }
-
-    public String generateTemporaryTableName(String tableName)
-    {
-        return generateTableName(tableName) + temporaryTableSuffix;
-    }
-
-    public enum SourceFormat
-    {
-        CSV("CSV"),
-        NEWLINE_DELIMITED_JSON("NEWLINE_DELIMITED_JSON");
-
-        private final String string;
-
-        SourceFormat(String string)
-        {
-            this.string = string;
-        }
-
-        public String getString()
-        {
-            return string;
-        }
-    }
-
-    public enum AuthMethod
-    {
-        private_key("private_key"),
-        compute_engine("compute_engine"),
-        json_key("json_key");
-
-        private final String string;
-
-        AuthMethod(String string)
-        {
-            this.string = string;
-        }
-
-        public String getString()
-        {
-            return string;
-        }
-    }
-
-    public enum Mode
-    {
-        append("append"),
-        delete_in_advance("delete_in_advance") {
-            @Override
-            public boolean isDeleteInAdvance()
-            {
-                return true;
-            }
-        },
-        replace("replace") {
-            @Override
-            public boolean isReplaceMode()
-            {
-                return true;
-            }
-        },
-        replace_backup("replace_backup") {
-            @Override
-            public boolean isReplaceMode()
-            {
-                return true;
-            }
-        };
-
-        private final String string;
-
-        Mode(String string)
-        {
-            this.string = string;
-        }
-
-        public String getString()
-        {
-            return string;
-        }
-        public boolean isReplaceMode()
-        {
-            return false;
-        }
-        public boolean isDeleteInAdvance()
-        {
-            return false;
-        }
-    }
-}