embulk-output-bigquery 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76284d5061944ac0c9f2f315cb46bc681b81dce0
-  data.tar.gz: f75c285ff585d5177273fd70897e320c6375c6f3
+  metadata.gz: 46c61dd1c73ff99c3c69bd217ca772f07b2e1127
+  data.tar.gz: ba184360972884260c1fe90264af7d5386791804
 SHA512:
-  metadata.gz: 7d4b83984a02db3af2dfb769f136d650c0458b5c3e9f0ab64823a558a8ca61885845a466133cc97c6092be60f05258d1e1d24ffd61c0fa78eb7018027d63a02c
-  data.tar.gz: 7e4296ff13ebd7c3c9ff68e6a1701a4e26214b47a08714fe4ae31889d919433ee3f42341ad65651a44e8a85d3cf43f8bc8a83238c0d67d44468e0f39b48940d5
+  metadata.gz: aa693e59cb4b45c2d43f07479f3d61e63242185be9964d4f00b83a4a784a0443ae270a63760f3f2f188e74deb77cbb94a89a18db49d2c5cd4621f18b73363ab3
+  data.tar.gz: 7c0ea783220de28befd7c565ff83ec5ff58f13af0db16b3d341a12c3e415adeacba375e5688a42fcbb26d0402a48071622ed5b161fa52fd08b1f56444faf66e1

data/README.md

@@ -1,17 +1,17 @@
 
 # embulk-output-bigquery
 
-[Embulk](https://github.com/embulk/embulk/) output plugin to load/insert data into [Google BigQuery](https://cloud.google.com/bigquery/) via [GCS(Google Cloud Storage)](https://cloud.google.com/storage/)
+[Embulk](https://github.com/embulk/embulk/) output plugin to load/insert data into [Google BigQuery](https://cloud.google.com/bigquery/)
 
 ## Overview
 
-load data into Google BigQuery as batch jobs via GCS for big amount of data
+load data into Google BigQuery as batch jobs for big amount of data
 https://developers.google.com/bigquery/loading-data-into-bigquery
 
 * **Plugin type**: output
 * **Resume supported**: no
 * **Cleanup supported**: no
-* **Dynamic table creating**: todo
+* **Dynamic table creating**: yes
 
 ### NOT IMPLEMENTED 
 * insert data over streaming inserts
@@ -30,32 +30,19 @@ OAuth flow for installed applications.
 - **sequence_format**: (string, optional, default is %03d.%02d)
 - **file_ext**: (string, required)
 - **source_format**: file type (NEWLINE_DELIMITED_JSON or CSV) (string, required, default is CSV)
-- **is_file_compressed**: upload file is gzip compressed or not. (boolean, optional, default is 1)
-- **bucket**: Google Cloud Storage output bucket name (string, required)
-- **remote_path**: folder name in GCS bucket (string, optional)
 - **project**: project_id (string, required)
 - **dataset**: dataset (string, required)
 - **table**: table name (string, required)
+- **auto_create_table**: (boolean, optional default is 0)
+- **schema_path**: (string, optional)
 - **application_name**: application name anything you like (string, optional)
-- **delete_from_local_when_upload_end**: (boolean, optional, default is 0)
-- **delete_from_bucket_when_job_end**: (boolean, optional, default is 0)
+- **delete_from_local_when_job_end**: (boolean, optional, default is 0)
 - **job_status_max_polling_time**: max job status polling time. (int, optional, default is 3600 sec)
 - **job_status_polling_interval**: job status polling interval. (int, optional, default is 10 sec)
 - **is_skip_job_result_check**: (boolean, optional, default is 0)
-
-## Support for Google BigQuery Quota policy
-embulk-output-bigquery support following [Google BigQuery Quota policy](https://cloud.google.com/bigquery/loading-data-into-bigquery#quota).
-
-* Supported
-  * Maximum size per load job: 1TB across all input files
-  * Maximum number of files per load job: 10,000
-    * embulk-output-bigquery divides a file into more than one job, like below.
-      * job1: file1(1GB) file2(1GB)...file10(1GB)
-      * job2: file11(1GB) file12(1GB)
-
-* Not Supported
-  * Daily limit: 1,000 load jobs per table per day (including failures)
-  * 10,000 load jobs per project per day (including failures)
+- **field_delimiter**: (string, optional, default is ",")
+- **max_bad_records**: (int, optional, default is 0)
+- **encoding**: (UTF-8 or ISO-8859-1) (string, optional, default is "UTF-8")
 
 ## Example
 
@@ -67,10 +54,7 @@ out:
   path_prefix: /path/to/output
   file_ext: csv.gz
   source_format: CSV
-  is_file_compressed: 1
   project: your-project-000
-  bucket: output_bucket_name
-  remote_path: folder_name
   dataset: your_dataset_name
   table: your_table_name
   formatter:
@@ -80,6 +64,21 @@ out:
   - {type: gzip}
 ```
 
+## Dynamic table creating
+
+When `auto_create_table` is set to true, try to create the table using BigQuery API.
+
+To describe the schema of the target table, please write schema path.
+
+`table` option accept [Time#strftime](http://ruby-doc.org/core-1.9.3/Time.html#method-i-strftime)
+format of ruby to construct table name.
+
+```
+auto_create_table: true
+table: table_%Y_%m
+schema_path: /path/to/schema.json
+```
+
 ## Build
 
 ```

data/build.gradle

@@ -15,16 +15,14 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.1"
+version = "0.1.2"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.5.1"
     provided "org.embulk:embulk-core:0.5.1"
 
     compile "com.google.http-client:google-http-client-jackson2:1.19.0"
-    compile ("com.google.apis:google-api-services-storage:v1-rev27-1.19.1") {exclude module: "guava-jdk5"}
     compile "com.google.apis:google-api-services-bigquery:v2-rev193-1.19.1"
-    compile "eu.medsea.mimeutil:mime-util:2.1.3"
 
     testCompile "junit:junit:4.+"
 }

data/src/main/java/org/embulk/output/BigqueryAuthentication.java

@@ -23,8 +23,6 @@ import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.InputStreamContent;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.storage.Storage;
-import com.google.api.services.storage.StorageScopes;
 import com.google.api.services.bigquery.Bigquery;
 import com.google.api.services.bigquery.BigqueryScopes;
 import com.google.api.services.bigquery.model.ProjectList;
@@ -64,7 +62,6 @@ public class BigqueryAuthentication
                 .setServiceAccountId(serviceAccountEmail)
                 .setServiceAccountScopes(
                         ImmutableList.of(
-                                BigqueryScopes.DEVSTORAGE_READ_WRITE,
                                 BigqueryScopes.BIGQUERY
                         )
                 )
@@ -87,13 +84,4 @@ public class BigqueryAuthentication
 
         return client;
     }
-
-    public Storage getGcsClient() throws IOException
-    {
-        Storage client = new Storage.Builder(httpTransport, jsonFactory, credentials)
-                .setApplicationName(applicationName)
-                .build();
-
-        return client;
-    }
 }

data/src/main/java/org/embulk/output/BigqueryOutputPlugin.java

@@ -13,6 +13,7 @@ import java.util.concurrent.TimeoutException;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import java.security.GeneralSecurityException;
+import org.jruby.embed.ScriptingContainer;
 
 import org.embulk.config.Config;
 import org.embulk.config.ConfigException;
@@ -59,10 +60,6 @@ public class BigqueryOutputPlugin
         @ConfigDefault("\"CSV\"")
         public String getSourceFormat();
 
-        @Config("is_file_compressed")
-        @ConfigDefault("true")
-        public boolean getIsFileCompressed();
-
         @Config("field_delimiter")
         @ConfigDefault("\",\"")
         public String getFieldDelimiter();
@@ -71,20 +68,13 @@ public class BigqueryOutputPlugin
         @ConfigDefault("0")
         public int getMaxBadrecords();
 
-        @Config("delete_from_local_when_upload_end")
-        @ConfigDefault("false")
-        public boolean getDeleteFromLocalWhenUploadEnd();
+        @Config("encoding")
+        @ConfigDefault("\"UTF-8\"")
+        public String getEncoding();
 
-        @Config("delete_from_bucket_when_job_end")
+        @Config("delete_from_local_when_job_end")
         @ConfigDefault("false")
-        public boolean getDeleteFromBucketWhenJobEnd();
-
-        @Config("bucket")
-        public String getBucket();
-
-        @Config("remote_path")
-        @ConfigDefault("null")
-        public Optional<String> getRemotePath();
+        public boolean getDeleteFromLocalWhenJobEnd();
 
         @Config("project")
         public String getProject();
@@ -117,7 +107,6 @@ public class BigqueryOutputPlugin
     }
 
     private final Logger log = Exec.getLogger(BigqueryOutputPlugin.class);
-    private static BigqueryGcsWriter bigQueryGcsWriter;
     private static BigqueryWriter bigQueryWriter;
 
     public ConfigDiff transaction(ConfigSource config, int taskCount,
@@ -126,33 +115,25 @@ public class BigqueryOutputPlugin
         final PluginTask task = config.loadConfig(PluginTask.class);
 
         try {
-            bigQueryGcsWriter = new BigqueryGcsWriter.Builder(task.getServiceAccountEmail())
-                    .setP12KeyFilePath(task.getP12KeyfilePath())
-                    .setApplicationName(task.getApplicationName())
-                    .setBucket(task.getBucket())
-                    .setSourceFormat(task.getSourceFormat())
-                    .setIsFileCompressed(task.getIsFileCompressed())
-                    .setDeleteFromBucketWhenJobEnd(task.getDeleteFromBucketWhenJobEnd())
-                    .build();
-
             bigQueryWriter = new BigqueryWriter.Builder(task.getServiceAccountEmail())
                     .setP12KeyFilePath(task.getP12KeyfilePath())
                     .setApplicationName(task.getApplicationName())
                     .setProject(task.getProject())
                     .setDataset(task.getDataset())
-                    .setTable(task.getTable())
+                    .setTable(generateTableName(task.getTable()))
                     .setAutoCreateTable(task.getAutoCreateTable())
                     .setSchemaPath(task.getSchemaPath())
-                    .setBucket(task.getBucket())
                     .setSourceFormat(task.getSourceFormat())
                     .setFieldDelimiter(task.getFieldDelimiter())
                     .setMaxBadrecords(task.getMaxBadrecords())
+                    .setEncoding(task.getEncoding())
                     .setJobStatusMaxPollingTime(task.getJobStatusMaxPollingTime())
                     .setJobStatusPollingInterval(task.getJobStatusPollingInterval())
                     .setIsSkipJobResultCheck(task.getIsSkipJobResultCheck())
                     .build();
+        } catch (FileNotFoundException ex) {
+            throw new ConfigException(ex);
         } catch (IOException | GeneralSecurityException ex) {
-            log.warn("Google Authentication was failed. Please Check your configurations.");
             throw new ConfigException(ex);
         }
         // non-retryable (non-idempotent) output:
@@ -165,19 +146,6 @@ public class BigqueryOutputPlugin
     {
         control.run(taskSource);
 
-        try {
-            bigQueryWriter.executeJob();
-            // TODO refactor
-            if (bigQueryGcsWriter.getDeleteFromBucketWhenJobEnd()) {
-                ArrayList<HashMap<String, String>> fileList = bigQueryWriter.getFileList();
-                for (HashMap<String, String> file : fileList) {
-                    bigQueryGcsWriter.deleteFile(file.get("remote_path"), file.get("file_name"));
-                }
-            }
-        } catch (IOException | TimeoutException | BigqueryWriter.JobFailedException ex) {
-            log.warn(ex.getMessage());
-            throw Throwables.propagate(ex);
-        }
         return Exec.newConfigDiff();
     }
 
@@ -196,7 +164,6 @@ public class BigqueryOutputPlugin
         final String pathPrefix = task.getPathPrefix();
         final String sequenceFormat = task.getSequenceFormat();
         final String pathSuffix = task.getFileNameExtension();
-        final Optional<String> remotePath = task.getRemotePath();
 
         return new TransactionalFileOutput() {
             private int fileIndex = 0;
@@ -217,7 +184,6 @@ public class BigqueryOutputPlugin
                     }
                     filePath = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + suffix;
                     file = new File(filePath);
-                    fileName = file.getName();
 
                     String parentPath = file.getParent();
                     File dir = new File(parentPath);
@@ -257,18 +223,15 @@ public class BigqueryOutputPlugin
             public void finish()
             {
                 closeFile();
-                if (fileName != null) {
-                    fileSize = file.length();
+                if (filePath != null) {
                     try {
-                        bigQueryGcsWriter.uploadFile(filePath, fileName, remotePath);
+                        bigQueryWriter.executeLoad(filePath);
 
-                        if (task.getDeleteFromLocalWhenUploadEnd()) {
+                        if (task.getDeleteFromLocalWhenJobEnd()) {
                             log.info(String.format("Delete local file [%s]", filePath));
                             file.delete();
                         }
-
-                        bigQueryWriter.addTask(remotePath, fileName, fileSize);
-                    } catch (IOException ex) {
+                    } catch (IOException | TimeoutException | BigqueryWriter.JobFailedException ex) {
                         throw Throwables.propagate(ex);
                     }
                 }
@@ -290,4 +253,13 @@ public class BigqueryOutputPlugin
             }
         };
     }
+
+    // Parse like "table_%Y_%m"(include pattern or not) format using Java is difficult. So use jRuby.
+    public String generateTableName(String tableName)
+    {
+        ScriptingContainer jruby = new ScriptingContainer();
+        Object result = jruby.runScriptlet("Time.now.strftime('" + tableName + "')");
+
+        return result.toString();
+    }
 }

data/src/main/java/org/embulk/output/BigqueryWriter.java

@@ -1,6 +1,11 @@
 package org.embulk.output;
 
+import java.io.File;
 import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.io.FileInputStream;
+import java.io.BufferedInputStream;
+import com.google.api.client.http.InputStreamContent;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Iterator;
@@ -11,14 +16,19 @@ import java.util.concurrent.TimeoutException;
 import org.apache.commons.lang3.StringUtils;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.base.Throwables;
 import java.security.GeneralSecurityException;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.core.type.TypeReference;
+
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
 
 import com.google.api.services.bigquery.Bigquery;
 import com.google.api.services.bigquery.BigqueryScopes;
 import com.google.api.services.bigquery.Bigquery.Datasets;
+import com.google.api.services.bigquery.Bigquery.Tables;
 import com.google.api.services.bigquery.Bigquery.Jobs.Insert;
 import com.google.api.services.bigquery.Bigquery.Jobs.GetQueryResults;
 import com.google.api.services.bigquery.model.Job;
@@ -28,11 +38,19 @@ import com.google.api.services.bigquery.model.JobStatus;
 import com.google.api.services.bigquery.model.JobStatistics;
 import com.google.api.services.bigquery.model.JobReference;
 import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
 import com.google.api.services.bigquery.model.TableSchema;
 import com.google.api.services.bigquery.model.TableReference;
 import com.google.api.services.bigquery.model.TableFieldSchema;
 import com.google.api.services.bigquery.model.TableCell;
 import com.google.api.services.bigquery.model.TableRow;
+import com.google.api.services.bigquery.model.ErrorProto;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+
+import com.google.api.client.googleapis.media.MediaHttpUploader;
+import com.google.api.client.googleapis.media.MediaHttpUploaderProgressListener;
+import com.google.api.client.googleapis.media.MediaHttpUploader.UploadState;
 
 public class BigqueryWriter
 {
@@ -43,43 +61,58 @@ public class BigqueryWriter
     private final String table;
     private final boolean autoCreateTable;
     private final Optional<String> schemaPath;
-    private final String bucket;
+    private final TableSchema tableSchema;
     private final String sourceFormat;
     private final String fieldDelimiter;
     private final int maxBadrecords;
+    private final String encoding;
     private final long jobStatusMaxPollingTime;
     private final long jobStatusPollingInterval;
     private final boolean isSkipJobResultCheck;
     private final Bigquery bigQueryClient;
-    private final EmbulkBigqueryTask writerTask;
 
-    public BigqueryWriter(Builder builder) throws IOException, GeneralSecurityException
+    public BigqueryWriter(Builder builder) throws FileNotFoundException, IOException, GeneralSecurityException
     {
         this.project = builder.project;
         this.dataset = builder.dataset;
         this.table = builder.table;
         this.autoCreateTable = builder.autoCreateTable;
         this.schemaPath = builder.schemaPath;
-        this.bucket = builder.bucket;
         this.sourceFormat = builder.sourceFormat.toUpperCase();
         this.fieldDelimiter = builder.fieldDelimiter;
         this.maxBadrecords = builder.maxBadrecords;
+        this.encoding = builder.encoding.toUpperCase();
         this.jobStatusMaxPollingTime = builder.jobStatusMaxPollingTime;
         this.jobStatusPollingInterval = builder.jobStatusPollingInterval;
         this.isSkipJobResultCheck = builder.isSkipJobResultCheck;
 
         BigqueryAuthentication auth = new BigqueryAuthentication(builder.serviceAccountEmail, builder.p12KeyFilePath, builder.applicationName);
         this.bigQueryClient = auth.getBigqueryClient();
-        this.writerTask = new EmbulkBigqueryTask();
+
+        checkConfig();
+        if (autoCreateTable) {
+            this.tableSchema = createTableSchema(builder.schemaPath);
+        } else {
+            this.tableSchema = null;
+        }
     }
 
     private String getJobStatus(JobReference jobRef) throws JobFailedException
     {
         try {
             Job job = bigQueryClient.jobs().get(project, jobRef.getJobId()).execute();
-            if (job.getStatus().getErrorResult() != null) {
-                throw new JobFailedException(String.format("Job failed. job id:[%s] reason:[%s] status:[FAILED]", jobRef.getJobId(), job.getStatus().getErrorResult().getMessage()));
+
+            ErrorProto fatalError = job.getStatus().getErrorResult();
+            if (fatalError != null) {
+                throw new JobFailedException(String.format("Job failed. job id:[%s] reason:[%s][%s] status:[FAILED]", jobRef.getJobId(), fatalError.getReason(), fatalError.getMessage()));
+            }
+            List<ErrorProto> errors = job.getStatus().getErrors();
+            if (errors != null) {
+                for (ErrorProto error : errors) {
+                    log.warn(String.format("Error: job id:[%s] reason[%s][%s] location:[%s]", jobRef.getJobId(), error.getReason(), error.getMessage(), error.getLocation()));
+                }
             }
+
             String jobStatus = job.getStatus().getState();
             if (jobStatus.equals("DONE")) {
                 JobStatistics statistics = job.getStatistics();
@@ -117,59 +150,68 @@ public class BigqueryWriter
         }
     }
 
-    public void executeJob() throws IOException, TimeoutException, JobFailedException
-    {
-        // TODO: refactor
-        ArrayList<ArrayList<HashMap<String, String>>> taskList = writerTask.createJobList();
-        for (ArrayList<HashMap<String, String>> task : taskList) {
-            Job job = createJob(task);
-            // TODO: multi-threading
-            new EmbulkBigqueryJob(job).call();
-        }
-    }
-
-    private Job createJob(ArrayList<HashMap<String, String>> task)
+    public void executeLoad(String localFilePath) throws GoogleJsonResponseException, IOException, TimeoutException, JobFailedException
     {
         log.info(String.format("Job preparing... project:%s dataset:%s table:%s", project, dataset, table));
 
         Job job = new Job();
+        JobReference jobRef = null;
         JobConfiguration jobConfig = new JobConfiguration();
         JobConfigurationLoad loadConfig = new JobConfigurationLoad();
         jobConfig.setLoad(loadConfig);
         job.setConfiguration(jobConfig);
 
         loadConfig.setAllowQuotedNewlines(false);
+        loadConfig.setEncoding(encoding);
+        loadConfig.setMaxBadRecords(maxBadrecords);
         if (sourceFormat.equals("NEWLINE_DELIMITED_JSON")) {
             loadConfig.setSourceFormat("NEWLINE_DELIMITED_JSON");
         } else {
             loadConfig.setFieldDelimiter(fieldDelimiter);
         }
+        loadConfig.setWriteDisposition("WRITE_APPEND");
         if (autoCreateTable) {
-            loadConfig.setSchema(getTableSchema());
-            loadConfig.setWriteDisposition("WRITE_EMPTY");
+            loadConfig.setSchema(tableSchema);
             loadConfig.setCreateDisposition("CREATE_IF_NEEDED");
-            log.info(String.format("table:[%s] will be create.", table));
+            log.info(String.format("table:[%s] will be create if not exists", table));
         } else {
-            loadConfig.setWriteDisposition("WRITE_APPEND");
             loadConfig.setCreateDisposition("CREATE_NEVER");
         }
-        loadConfig.setMaxBadRecords(maxBadrecords);
 
-        List<String> sources = new ArrayList<String>();
-        for (HashMap<String, String> file : task) {
-            String sourceFile;
-            String remotePath = getRemotePath(file.get("remote_path"), file.get("file_name"));
-            sourceFile = "gs://" + remotePath;
-            log.info(String.format("Add source file to job [%s]", sourceFile));
-            sources.add(sourceFile);
-        }
-        loadConfig.setSourceUris(sources);
-        loadConfig.setDestinationTable(getTableReference());
+        loadConfig.setDestinationTable(createTableReference());
+
+        File file = new File(localFilePath);
+        InputStreamContent mediaContent = new InputStreamContent("application/octet-stream",
+                new BufferedInputStream(
+                        new FileInputStream(file)));
+        mediaContent.setLength(file.length());
 
-        return job;
+        Insert insert = bigQueryClient.jobs().insert(project, job, mediaContent);
+        insert.setProjectId(project);
+        insert.setDisableGZipContent(true);
+
+        // @see https://code.google.com/p/google-api-java-client/wiki/MediaUpload
+        UploadProgressListener listner = new UploadProgressListener();
+        listner.setFileName(localFilePath);
+        insert.getMediaHttpUploader()
+                .setProgressListener(listner)
+                .setDirectUploadEnabled(false);
+
+        try {
+            jobRef = insert.execute().getJobReference();
+        } catch (Exception ex) {
+            log.warn("Job execution was failed. Please check your settings or data... like data matches schema");
+            throw Throwables.propagate(ex);
+        }
+        log.info(String.format("Job executed. job id:[%s] file:[%s]", jobRef.getJobId(), localFilePath));
+        if (isSkipJobResultCheck) {
+            log.info(String.format("Skip job status check. job id:[%s]", jobRef.getJobId()));
+        } else {
+            getJobStatusUntilDone(jobRef);
+        }
     }
 
-    private TableReference getTableReference()
+    private TableReference createTableReference()
     {
         return new TableReference()
                 .setProjectId(project)
@@ -177,135 +219,78 @@ public class BigqueryWriter
                 .setTableId(table);
     }
 
-    private TableSchema getTableSchema()
+    private TableSchema createTableSchema(Optional<String> schemaPath) throws FileNotFoundException, IOException
     {
-        TableSchema tableSchema = new TableSchema();
-        List<TableFieldSchema> fields = new ArrayList<TableFieldSchema>();
-        TableFieldSchema tableField;
-        // TODO import from json file
-        /*
-        for () {
-            tableField = new TableFieldSchema()
-                    .setName(name)
-                    .setType(type);
-            fields.add(tableField);
+        String path = schemaPath.orNull();
+        File file = new File(path);
+        FileInputStream stream = null;
+        try {
+            stream = new FileInputStream(file);
+            ObjectMapper mapper = new ObjectMapper();
+            List<TableFieldSchema> fields = mapper.readValue(stream, new TypeReference<List<TableFieldSchema>>() {});
+            TableSchema tableSchema = new TableSchema().setFields(fields);
+            return tableSchema;
+        } finally {
+            if (stream != null) {
+                stream.close();
+            }
         }
-        */
-
-        tableSchema.setFields(fields);
-        return tableSchema;
     }
 
-    private String getRemotePath(String remotePath, String fileName)
+    public boolean isExistTable(String tableName) throws IOException
     {
-        String[] pathList = StringUtils.split(remotePath, '/');
-        String path;
-        if (remotePath.isEmpty()) {
-            path = bucket + "/" + fileName;
-        } else {
-            path = bucket + "/" + StringUtils.join(pathList) + "/" + fileName;
+        Tables tableRequest = bigQueryClient.tables();
+        try {
+            Table tableData = tableRequest.get(project, dataset, tableName).execute();
+        } catch (GoogleJsonResponseException ex) {
+            return false;
         }
-        return path;
-    }
-
-    public void addTask(Optional<String> remotePath, String fileName, long fileSize)
-    {
-        writerTask.addTaskFile(remotePath, fileName, fileSize);
-    }
-
-    public ArrayList<HashMap<String, String>> getFileList()
-    {
-        return writerTask.getFileList();
+        return true;
     }
 
-    private class EmbulkBigqueryJob implements Callable<Void>
+    public void checkConfig() throws FileNotFoundException, IOException
     {
-        private final Job job;
-
-        public EmbulkBigqueryJob(Job job)
-        {
-            this.job = job;
-        }
-
-        public Void call() throws IOException, TimeoutException, JobFailedException
-        {
-            Insert insert = bigQueryClient.jobs().insert(project, job);
-            insert.setProjectId(project);
-            JobReference jobRef = insert.execute().getJobReference();
-            log.info(String.format("Job executed. job id:[%s]", jobRef.getJobId()));
-            if (isSkipJobResultCheck) {
-                log.info(String.format("Skip job status check. job id:[%s]", jobRef.getJobId()));
+        if (autoCreateTable) {
+            if (!schemaPath.isPresent()) {
+                throw new IOException("schema_path is empty");
             } else {
-                getJobStatusUntilDone(jobRef);
+                File file = new File(schemaPath.orNull());
+                if (!file.exists()) {
+                    throw new FileNotFoundException("Can not load schema file.");
+                }
+            }
+        } else {
+            if (!isExistTable(table)) {
+                throw new IOException(String.format("table [%s] is not exists", table));
             }
-            return null;
         }
     }
 
-    private class EmbulkBigqueryTask
+    private class UploadProgressListener implements MediaHttpUploaderProgressListener
     {
-        // https://cloud.google.com/bigquery/loading-data-into-bigquery#quota
-        private final long MAX_SIZE_PER_LOAD_JOB = 1000 * 1024 * 1024 * 1024L; // 1TB
-        private final int MAX_NUMBER_OF_FILES_PER_LOAD_JOB = 10000;
+        private String fileName;
 
-        private final ArrayList<HashMap<String, String>> taskList = new ArrayList<HashMap<String, String>>();
-        private final ArrayList<ArrayList<HashMap<String, String>>> jobList = new ArrayList<ArrayList<HashMap<String, String>>>();
-
-        public void addTaskFile(Optional<String> remotePath, String fileName, long fileSize)
+        @Override
+        public void progressChanged(MediaHttpUploader uploader) throws IOException
         {
-            HashMap<String, String> task = new HashMap<String, String>();
-            if (remotePath.isPresent()) {
-                task.put("remote_path", remotePath.get());
-            } else {
-                task.put("remote_path", "");
-            }
-            task.put("file_name", fileName);
-            task.put("file_size", String.valueOf(fileSize));
-            taskList.add(task);
-        }
-
-        public ArrayList<ArrayList<HashMap<String, String>>> createJobList()
-        {
-            long currentBundleSize = 0;
-            int currentFileCount = 0;
-            ArrayList<HashMap<String, String>> job = new ArrayList<HashMap<String, String>>();
-            for (HashMap<String, String> task : taskList) {
-                boolean isNeedNextJobList = false;
-                long fileSize = Long.valueOf(task.get("file_size")).longValue();
-
-                if (currentBundleSize + fileSize > MAX_SIZE_PER_LOAD_JOB) {
-                    isNeedNextJobList = true;
-                }
-
-                if (currentFileCount >= MAX_NUMBER_OF_FILES_PER_LOAD_JOB) {
-                    isNeedNextJobList = true;
-                }
-
-                if (isNeedNextJobList) {
-                    jobList.add(job);
-                    job = new ArrayList<HashMap<String, String>>();
-                    job.add(task);
-                    currentBundleSize = 0;
-                } else {
-                    job.add(task);
-                }
-                currentBundleSize += fileSize;
-                currentFileCount++;
-
-                log.debug(String.format("currentBundleSize:%s currentFileCount:%s", currentBundleSize, currentFileCount));
-                log.debug(String.format("fileSize:%s, MAX_SIZE_PER_LOAD_JOB:%s MAX_NUMBER_OF_FILES_PER_LOAD_JOB:%s",
-                        fileSize, MAX_SIZE_PER_LOAD_JOB, MAX_NUMBER_OF_FILES_PER_LOAD_JOB));
-
-            }
-            if (job.size() > 0) {
-                jobList.add(job);
+            switch (uploader.getUploadState()) {
+                case INITIATION_STARTED:
+                    log.info(String.format("Upload start [%s]", fileName));
+                    break;
+                case INITIATION_COMPLETE:
+                    //log.info(String.format("Upload initiation completed file [%s]", fileName));
+                    break;
+                case MEDIA_IN_PROGRESS:
+                    log.debug(String.format("Uploading [%s] progress %3.0f", fileName, uploader.getProgress() * 100) + "%");
+                    break;
+                case MEDIA_COMPLETE:
+                    log.info(String.format("Upload completed [%s]", fileName));
             }
-            return jobList;
         }
 
-        public ArrayList<HashMap<String, String>> getFileList()
+        public void setFileName(String fileName)
         {
-            return taskList;
+            this.fileName = fileName;
         }
     }
 
@@ -319,10 +304,10 @@ public class BigqueryWriter
         private String table;
         private boolean autoCreateTable;
         private Optional<String> schemaPath;
-        private String bucket;
         private String sourceFormat;
         private String fieldDelimiter;
         private int maxBadrecords;
+        private String encoding;
         private int jobStatusMaxPollingTime;
         private int jobStatusPollingInterval;
         private boolean isSkipJobResultCheck;
@@ -375,12 +360,6 @@ public class BigqueryWriter
             return this;
         }
 
-        public Builder setBucket(String bucket)
-        {
-            this.bucket = bucket;
-            return this;
-        }
-
         public Builder setSourceFormat(String sourceFormat)
         {
             this.sourceFormat = sourceFormat;
@@ -399,6 +378,12 @@ public class BigqueryWriter
             return this;
         }
 
+        public Builder setEncoding(String encoding)
+        {
+            this.encoding = encoding;
+            return this;
+        }
+
         public Builder setJobStatusMaxPollingTime(int jobStatusMaxPollingTime)
         {
             this.jobStatusMaxPollingTime = jobStatusMaxPollingTime;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-17 00:00:00.000000000 Z
+date: 2015-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -56,19 +56,16 @@ files:
 - lib/embulk/output/bigquery.rb
 - settings.gradle
 - src/main/java/org/embulk/output/BigqueryAuthentication.java
-- src/main/java/org/embulk/output/BigqueryGcsWriter.java
 - src/main/java/org/embulk/output/BigqueryOutputPlugin.java
 - src/main/java/org/embulk/output/BigqueryWriter.java
 - src/test/java/org/embulk/output/TestBigqueryAuthentication.java
-- src/test/java/org/embulk/output/TestBigqueryGcsWriter.java
 - src/test/java/org/embulk/output/TestBigqueryOutputPlugin.java
 - src/test/java/org/embulk/output/TestBigqueryWriter.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-output-bigquery-0.1.1.jar
+- classpath/embulk-output-bigquery-0.1.2.jar
 - classpath/google-api-client-1.19.1.jar
 - classpath/google-api-services-bigquery-v2-rev193-1.19.1.jar
-- classpath/google-api-services-storage-v1-rev27-1.19.1.jar
 - classpath/google-http-client-1.19.0.jar
 - classpath/google-http-client-jackson2-1.19.0.jar
 - classpath/google-oauth-client-1.19.0.jar
@@ -76,7 +73,6 @@ files:
 - classpath/httpclient-4.0.1.jar
 - classpath/httpcore-4.0.1.jar
 - classpath/jsr305-1.3.9.jar
-- classpath/mime-util-2.1.3.jar
 homepage: https://github.com/sakama/embulk-output-bigquery
 licenses:
 - Apache-2.0

data/src/main/java/org/embulk/output/BigqueryGcsWriter.java

@@ -1,201 +0,0 @@
-package org.embulk.output;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.IllegalFormatException;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
-//import eu.medsea.mimeutil.MimeType;
-//import eu.medsea.mimeutil.MimeUtil;
-//import eu.medsea.mimeutil.detector.MimeDetector;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.codec.binary.Base64;
-import java.security.GeneralSecurityException;
-
-import org.embulk.spi.Exec;
-import org.slf4j.Logger;
-
-import com.google.api.services.storage.Storage;
-import com.google.api.services.storage.StorageScopes;
-import com.google.api.services.storage.model.Bucket;
-import com.google.api.services.storage.model.Objects;
-import com.google.api.services.storage.model.StorageObject;
-
-import com.google.api.client.http.InputStreamContent;
-
-public class BigqueryGcsWriter
-{
-
-    private final Logger log = Exec.getLogger(BigqueryGcsWriter.class);
-    private final String bucket;
-    private final String sourceFormat;
-    private final boolean isFileCompressed;
-    private final boolean deleteFromBucketWhenJobEnd;
-    private Storage storageClient;
-
-    public BigqueryGcsWriter(Builder builder) throws IOException, GeneralSecurityException
-    {
-        this.bucket = builder.bucket;
-        this.sourceFormat = builder.sourceFormat.toUpperCase();
-        this.isFileCompressed = builder.isFileCompressed;
-        this.deleteFromBucketWhenJobEnd = builder.deleteFromBucketWhenJobEnd;
-
-        BigqueryAuthentication auth = new BigqueryAuthentication(builder.serviceAccountEmail, builder.p12KeyFilePath, builder.applicationName);
-        this.storageClient = auth.getGcsClient();
-    }
-
-    public void uploadFile(String localFilePath, String fileName, Optional<String> remotePath) throws IOException
-    {
-        FileInputStream stream = null;
-
-        try {
-            String path;
-            if (remotePath.isPresent()) {
-                path = remotePath.get();
-            } else {
-                path = "";
-            }
-            String gcsPath = getRemotePath(path, fileName);
-            StorageObject objectMetadata = new StorageObject().setName(gcsPath);
-            log.info(String.format("Uploading file [%s] to [gs://%s/%s]", localFilePath, bucket, gcsPath));
-
-            File file = new File(localFilePath);
-            stream = new FileInputStream(file);
-            InputStreamContent content = new InputStreamContent(getContentType(), stream);
-            Storage.Objects.Insert insertObject = storageClient.objects().insert(bucket, objectMetadata, content);
-            insertObject.setDisableGZipContent(true);
-
-            StorageObject response = insertObject.execute();
-            log.info(String.format("Upload completed [%s] to [gs://%s/%s]", localFilePath, bucket, gcsPath));
-        } finally {
-            stream.close();
-        }
-    }
-
-    private String getRemotePath(String remotePath, String fileName)
-    {
-        if (remotePath.isEmpty()) {
-            return fileName;
-        }
-        String[] pathList = StringUtils.split(remotePath, '/');
-        String path = StringUtils.join(pathList) + "/";
-        if (!path.endsWith("/")) {
-            path = path + "/";
-        }
-        return path + fileName;
-    }
-
-    public void deleteFile(String remotePath, String fileName) throws IOException
-    {
-        String path = getRemotePath(remotePath, fileName);
-        storageClient.objects().delete(bucket, path).execute();
-        log.info(String.format("Delete remote file [gs://%s/%s]", bucket, path));
-    }
-
-    public boolean getDeleteFromBucketWhenJobEnd()
-    {
-        return this.deleteFromBucketWhenJobEnd;
-    }
-
-    private String getContentType()
-    {
-        if (isFileCompressed) {
-            return "application/x-gzip";
-        } else {
-            if (sourceFormat.equals("NEWLINE_DELIMITED_JSON)")) {
-                return "application/json";
-            } else {
-                return "text/csv";
-            }
-        }
-    }
-
-    /*
-    private void registerMimeDetector()
-    {
-        String mimeDetector = "eu.medsea.mimeutil.detector.MagicMimeMimeDetector";
-        MimeDetector registeredMimeDetector = MimeUtil.getMimeDetector(mimeDetector);
-        MimeUtil.registerMimeDetector(mimeDetector);
-    }
-
-    public String detectMimeType(File file)
-    {
-        try {
-            Collection<?> mimeTypes = MimeUtil.getMimeTypes(file);
-            if (!mimeTypes.isEmpty()) {
-                Iterator<?> iterator = mimeTypes.iterator();
-                MimeType mimeType = (MimeType) iterator.next();
-                return mimeType.getMediaType() + "/" + mimeType.getSubType();
-            }
-        } catch (Exception ex) {
-        }
-        return "application/octet-stream";
-    }
-    */
-
-    public static class Builder
-    {
-        private final String serviceAccountEmail;
-        private String p12KeyFilePath;
-        private String applicationName;
-        private String bucket;
-        private String sourceFormat;
-        private boolean isFileCompressed;
-        private boolean deleteFromBucketWhenJobEnd;
-        private boolean enableMd5hashCheck;
-
-        public Builder(String serviceAccountEmail)
-        {
-            this.serviceAccountEmail = serviceAccountEmail;
-        }
-
-        public Builder setP12KeyFilePath(String p12KeyFilePath)
-        {
-            this.p12KeyFilePath = p12KeyFilePath;
-            return this;
-        }
-
-        public Builder setApplicationName(String applicationName)
-        {
-            this.applicationName = applicationName;
-            return this;
-        }
-
-        public Builder setBucket(String bucket)
-        {
-            this.bucket = bucket;
-            return this;
-        }
-
-        public Builder setSourceFormat(String sourceFormat)
-        {
-            this.sourceFormat = sourceFormat;
-            return this;
-        }
-
-        public Builder setIsFileCompressed(boolean isFileCompressed)
-        {
-            this.isFileCompressed = isFileCompressed;
-            return this;
-        }
-
-        public Builder setDeleteFromBucketWhenJobEnd(boolean deleteFromBucketWhenJobEnd)
-        {
-            this.deleteFromBucketWhenJobEnd = deleteFromBucketWhenJobEnd;
-            return this;
-        }
-
-        public BigqueryGcsWriter build() throws IOException, GeneralSecurityException
-        {
-            return new BigqueryGcsWriter(this);
-        }
-    }
-}

data/src/test/java/org/embulk/output/TestBigqueryGcsWriter.java

@@ -1,5 +0,0 @@
-package org.embulk.output;
-
-public class TestBigqueryGcsWriter
-{
-}