embulk-output-bigquery 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a814af5998dfc7f29f8cac8d06b49ec0b6713ff6
-  data.tar.gz: 0381957abca63f41d93c768c1b1f0f1ac62a4339
+  metadata.gz: 48a9a0add9223ccfca3e1e48f360ebf38cfe08d5
+  data.tar.gz: b6d9d0b0635c9d0728238873094122b4a72648de
 SHA512:
-  metadata.gz: ab1ee4e8bf996e9540c69352881449b29e1818a5e4d23babb8df6f47fe08c1cadd80ec373ed271c6e919ebd13f009bdba56e3d13ea2aa27085ef8e45e7f16c8b
-  data.tar.gz: 1274ac1eed4de4d046229a724a718d32e33a8c16d17844ffb4a4f94e4dc97259d5f6902222a6c34d4bba1525a6e184070d84f05bb192cbccc8b72c4166287b84
+  metadata.gz: 7ee81bbbc5b65c34d86014e3da5a3d848ded06766a3b58e92f5918070cc89eadd8740cf16f2b36c864c7a50f5476a0be2460fe2b722f2edfeb23f612f721df91
+  data.tar.gz: 605003c7364982e0f4fc391ef51517ef16b201b0dc0c3f43833f1cd98a5110b7ae2476f64ebbf3d9284e3b781a2ed85b68855d3b585cdda4b68e72f223974b14

data/README.md

@@ -30,14 +30,14 @@ OAuth flow for installed applications.
 |:--------------------------|:------------|:-----------|:-------------|:-----------------------|
 |  auth_method              | string      | optional   | "private_key"  | `private_key` or `compute_engine`
 |  service_account_email    | string      | required when auth_method is private_key  |   | Your Google service account email
-|  p12_keyfile_path         | string      | required when auth_method is private_key   |   | Fullpath of private key in P12(PKCS12) format |
+|  p12_keyfile              | string      | required when auth_method is private_key   |   | Fullpath of private key in P12(PKCS12) format |
 |  sequence_format          | string      | optional   | %03d.%02d      |  |
 |  file_ext                 | string      | optional   |                | e.g. ".csv.gz" ".json.gz" |
 |  project                  | string      | required   |                | project_id |
 |  dataset                  | string      | required   |                | dataset |
 |  table                    | string      | required   |                | table name |
 |  auto_create_table        | boolean     | optional   | 0              | [See below](#dynamic-table-creating) |
-|  schema_path              | string      | optional   |                | /path/to/schema.json |
+|  schema_file              | string      | optional   |                | /path/to/schema.json |
 |  prevent_duplicate_insert | boolean     | optional   | 0              | [See below](#data-consistency) |
 |  delete_from_local_when_job_end | boolean     | optional   | 0            | If set to true, delete local file when job is end |
 |  job_status_max_polling_time    | int         | optional   | 3600 sec     | Max job status polling time |
@@ -53,7 +53,7 @@ Following options are same as [bq command-line tools](https://cloud.google.com/b
 |:--------------------------|:------------|:-----------|:-------------|:-----------------------|
 |  source_format            | string      | required   | "CSV"          | File type (`NEWLINE_DELIMITED_JSON` or `CSV`) |
 |  max_bad_records          | int         | optional   | 0            | |
-|  field_delimiter          | string      | optional   | ","          | |
+|  field_delimiter          | char        | optional   | ","          |  |
 |  encoding                 | string      | optional   | "UTF-8"      | `UTF-8` or `ISO-8859-1` |
 |  ignore_unknown_values    | boolean     | optional   | 0            | |
 |  allow_quoted_newlines    | boolean     | optional   | 0            | Set true, if data contains newline characters. It may cause slow procsssing |
@@ -65,7 +65,7 @@ out:
   type: bigquery
   auth_method: private_key   # default
   service_account_email: ABCXYZ123ABCXYZ123.gserviceaccount.com
-  p12_keyfile_path: /path/to/p12_keyfile.p12
+  p12_keyfile: /path/to/p12_keyfile.p12
   path_prefix: /path/to/output
   file_ext: csv.gz
   source_format: CSV
@@ -130,14 +130,14 @@ out:
   type: bigquery
   auto_create_table: true
   table: table_%Y_%m
-  schema_path: /path/to/schema.json
+  schema_file: /path/to/schema.json
 ```
 
 ### Data Consistency
 
 When `prevent_duplicate_insert` is set to true, embulk-output-bigquery generate job ID from md5 hash of file  and other options to prevent duplicate data insertion.
 
-`job ID = md5(md5(file) + dataset + table + schema + source_format + file_delimiter + max_bad_records + encoding + ignore_unknown_values)`
+`job ID = md5(md5(file) + dataset + table + schema + source_format + file_delimiter + max_bad_records + encoding + ignore_unknown_values + allow_quoted_newlines)`
 
 [job ID must be unique(including failures)](https://cloud.google.com/bigquery/loading-data-into-bigquery#consistency). So same data can't insert with same settings.
 

data/build.gradle

@@ -15,11 +15,11 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.7"
+version = "0.1.8"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.6.8"
-    provided "org.embulk:embulk-core:0.6.8"
+    compile  "org.embulk:embulk-core:0.6.22"
+    provided "org.embulk:embulk-core:0.6.22"
 
     compile "com.google.http-client:google-http-client-jackson2:1.20.0"
     compile "com.google.apis:google-api-services-bigquery:v2-rev205-1.20.0"
@@ -49,7 +49,7 @@ Gem::Specification.new do |spec|
   spec.description   = %[Embulk plugin that insert records to Google BigQuery.]
   spec.email         = ["satoshiakama@gmail.com"]
   spec.licenses      = ["Apache-2.0"]
-  spec.homepage      = "https://github.com/sakama/embulk-output-bigquery"
+  spec.homepage      = "https://github.com/embulk/embulk-output-bigquery"
 
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")

data/src/main/java/org/embulk/output/BigqueryOutputPlugin.java

@@ -5,9 +5,11 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.BufferedOutputStream;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.security.NoSuchAlgorithmException;
 import java.util.List;
 import java.util.concurrent.TimeoutException;
+import com.google.common.base.Function;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import java.security.GeneralSecurityException;
@@ -21,6 +23,7 @@ import org.embulk.config.ConfigDiff;
 import org.embulk.config.CommitReport;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
+import org.embulk.spi.unit.LocalFile;
 import org.embulk.spi.Buffer;
 import org.embulk.spi.FileOutputPlugin;
 import org.embulk.spi.TransactionalFileOutput;
@@ -36,114 +39,147 @@ public class BigqueryOutputPlugin
     {
         @Config("auth_method")
         @ConfigDefault("\"private_key\"")
-        public String getAuthMethod();
+        AuthMethod getAuthMethod();
 
         @Config("service_account_email")
         @ConfigDefault("null")
-        public Optional<String> getServiceAccountEmail();
+        Optional<String> getServiceAccountEmail();
 
+        // kept for backward compatibility
         @Config("p12_keyfile_path")
         @ConfigDefault("null")
-        public Optional<String> getP12KeyfilePath();
+        Optional<String> getP12KeyfilePath();
+
+        @Config("p12_keyfile")
+        @ConfigDefault("null")
+        Optional<LocalFile> getP12Keyfile();
+        void setP12Keyfile(Optional<LocalFile> p12Keyfile);
 
         @Config("application_name")
         @ConfigDefault("\"Embulk BigQuery plugin\"")
-        public String getApplicationName();
+        String getApplicationName();
 
         @Config("path_prefix")
-        public String getPathPrefix();
+        String getPathPrefix();
 
         @Config("sequence_format")
         @ConfigDefault("\".%03d.%02d\"")
-        public String getSequenceFormat();
+        String getSequenceFormat();
 
         @Config("file_ext")
-        public String getFileNameExtension();
+        String getFileNameExtension();
 
         @Config("source_format")
         @ConfigDefault("\"CSV\"")
-        public String getSourceFormat();
+        SourceFormat getSourceFormat();
 
         @Config("field_delimiter")
         @ConfigDefault("\",\"")
-        public String getFieldDelimiter();
+        char getFieldDelimiter();
 
         @Config("max_bad_records")
         @ConfigDefault("0")
-        public int getMaxBadrecords();
+        int getMaxBadrecords();
 
         @Config("encoding")
         @ConfigDefault("\"UTF-8\"")
-        public String getEncoding();
+        Charset getEncoding();
 
         @Config("delete_from_local_when_job_end")
         @ConfigDefault("false")
-        public boolean getDeleteFromLocalWhenJobEnd();
+        boolean getDeleteFromLocalWhenJobEnd();
 
         @Config("project")
-        public String getProject();
+        String getProject();
 
         @Config("dataset")
-        public String getDataset();
+        String getDataset();
 
         @Config("table")
-        public String getTable();
+        String getTable();
 
         @Config("auto_create_table")
         @ConfigDefault("false")
-        public boolean getAutoCreateTable();
+        boolean getAutoCreateTable();
 
+        // kept for backward compatibility
         @Config("schema_path")
         @ConfigDefault("null")
-        public Optional<String> getSchemaPath();
+        Optional<String> getSchemaPath();
+
+        @Config("schema_file")
+        @ConfigDefault("null")
+        Optional<LocalFile> getSchemaFile();
+        void setSchemaFile(Optional<LocalFile> schemaFile);
 
         @Config("prevent_duplicate_insert")
         @ConfigDefault("false")
-        public boolean getPreventDuplicateInsert();
+        boolean getPreventDuplicateInsert();
 
         @Config("job_status_max_polling_time")
         @ConfigDefault("3600")
-        public int getJobStatusMaxPollingTime();
+        int getJobStatusMaxPollingTime();
 
         @Config("job_status_polling_interval")
         @ConfigDefault("10")
-        public int getJobStatusPollingInterval();
+        int getJobStatusPollingInterval();
 
         @Config("is_skip_job_result_check")
         @ConfigDefault("false")
-        public boolean getIsSkipJobResultCheck();
+        boolean getIsSkipJobResultCheck();
 
         @Config("ignore_unknown_values")
         @ConfigDefault("false")
-        public boolean getIgnoreUnknownValues();
+        boolean getIgnoreUnknownValues();
 
         @Config("allow_quoted_newlines")
         @ConfigDefault("false")
-        public boolean getAllowQuotedNewlines();
+        boolean getAllowQuotedNewlines();
     }
 
     private final Logger log = Exec.getLogger(BigqueryOutputPlugin.class);
     private static BigqueryWriter bigQueryWriter;
 
+    @Override
     public ConfigDiff transaction(ConfigSource config, int taskCount,
                                   FileOutputPlugin.Control control)
     {
         final PluginTask task = config.loadConfig(PluginTask.class);
 
+        if (task.getP12KeyfilePath().isPresent()) {
+            if (task.getP12Keyfile().isPresent()) {
+                throw new ConfigException("Setting both p12_keyfile_path and p12_keyfile is invalid");
+            }
+            try {
+                task.setP12Keyfile(Optional.of(LocalFile.of(task.getP12KeyfilePath().get())));
+            } catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        if (task.getSchemaPath().isPresent()) {
+            if (task.getSchemaFile().isPresent()) {
+                throw new ConfigException("Setting both p12_keyfile_path and p12_keyfile is invalid");
+            }
+            try {
+                task.setSchemaFile(Optional.of(LocalFile.of(task.getSchemaPath().get())));
+            } catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
         try {
-            bigQueryWriter = new BigqueryWriter.Builder(task.getAuthMethod())
-                    .setServiceAccountEmail(task.getServiceAccountEmail())
-                    .setP12KeyFilePath(task.getP12KeyfilePath())
-                    .setApplicationName(task.getApplicationName())
-                    .setProject(task.getProject())
-                    .setDataset(task.getDataset())
-                    .setTable(generateTableName(task.getTable()))
+            bigQueryWriter = new BigqueryWriter.Builder (
+                    task.getAuthMethod().getString(),
+                    task.getServiceAccountEmail(),
+                    task.getP12Keyfile().transform(localFileToPathString()),
+                    task.getApplicationName())
                     .setAutoCreateTable(task.getAutoCreateTable())
-                    .setSchemaPath(task.getSchemaPath())
-                    .setSourceFormat(task.getSourceFormat())
-                    .setFieldDelimiter(task.getFieldDelimiter())
-                    .setMaxBadrecords(task.getMaxBadrecords())
-                    .setEncoding(task.getEncoding())
+                    .setSchemaPath(task.getSchemaFile().transform(localFileToPathString()))
+                    .setSourceFormat(task.getSourceFormat().getString())
+                    .setFieldDelimiter(String.valueOf(task.getFieldDelimiter()))
+                    .setMaxBadRecords(task.getMaxBadrecords())
+                    .setEncoding(String.valueOf(task.getEncoding()))
                     .setPreventDuplicateInsert(task.getPreventDuplicateInsert())
                     .setJobStatusMaxPollingTime(task.getJobStatusMaxPollingTime())
                     .setJobStatusPollingInterval(task.getJobStatusPollingInterval())
@@ -151,8 +187,9 @@ public class BigqueryOutputPlugin
                     .setIgnoreUnknownValues(task.getIgnoreUnknownValues())
                     .setAllowQuotedNewlines(task.getAllowQuotedNewlines())
                     .build();
-        } catch (FileNotFoundException ex) {
-            throw new ConfigException(ex);
+
+            bigQueryWriter.checkConfig(task.getProject(), task.getDataset(), task.getTable());
+
         } catch (IOException | GeneralSecurityException ex) {
             throw new ConfigException(ex);
         }
@@ -160,6 +197,7 @@ public class BigqueryOutputPlugin
         return resume(task.dump(), taskCount, control);
     }
 
+    @Override
     public ConfigDiff resume(TaskSource taskSource,
                              int taskCount,
                              FileOutputPlugin.Control control)
@@ -176,6 +214,17 @@ public class BigqueryOutputPlugin
     {
     }
 
+    private Function<LocalFile, String> localFileToPathString()
+    {
+        return new Function<LocalFile, String>()
+        {
+            public String apply(LocalFile file)
+            {
+                return file.getPath().toString();
+            }
+        };
+    }
+
     @Override
     public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
     {
@@ -186,6 +235,11 @@ public class BigqueryOutputPlugin
         final String pathSuffix = task.getFileNameExtension();
 
         return new TransactionalFileOutput() {
+            private final String project = task.getProject();
+            private final String dataset = task.getDataset();
+            private final String table = generateTableName(task.getTable());
+            private final boolean deleteFromLocalWhenJobEnd = task.getDeleteFromLocalWhenJobEnd();
+
             private int fileIndex = 0;
             private BufferedOutputStream output = null;
             private File file;
@@ -243,9 +297,9 @@ public class BigqueryOutputPlugin
                 closeFile();
                 if (filePath != null) {
                     try {
-                        bigQueryWriter.executeLoad(filePath);
+                        bigQueryWriter.executeLoad(project, dataset, table, filePath);
 
-                        if (task.getDeleteFromLocalWhenJobEnd()) {
+                        if (deleteFromLocalWhenJobEnd) {
                             log.info(String.format("Delete local file [%s]", filePath));
                             file.delete();
                         }
@@ -281,4 +335,40 @@ public class BigqueryOutputPlugin
 
         return result.toString();
     }
-}
+
+    public enum SourceFormat
+    {
+        CSV("CSV"),
+        NEWLINE_DELIMITED_JSON("NEWLINE_DELIMITED_JSON");
+
+        private final String string;
+
+        SourceFormat(String string)
+        {
+            this.string = string;
+        }
+
+        public String getString()
+        {
+            return string;
+        }
+    }
+
+    public enum AuthMethod
+    {
+        private_key("private_key"),
+        compute_engine("compute_engine");
+
+        private final String string;
+
+        AuthMethod(String string)
+        {
+            this.string = string;
+        }
+
+        public String getString()
+        {
+            return string;
+        }
+    }
+}

data/src/main/java/org/embulk/output/BigqueryWriter.java

@@ -6,17 +6,17 @@ import java.io.FileNotFoundException;
 import java.io.FileInputStream;
 import java.io.BufferedInputStream;
 import com.google.api.client.http.InputStreamContent;
+
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.List;
 import java.util.concurrent.TimeoutException;
 import com.google.common.base.Optional;
-import com.google.api.client.util.Base64;
-import com.google.common.base.Throwables;
 import java.security.GeneralSecurityException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.core.type.TypeReference;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.commons.codec.binary.Hex;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
@@ -40,17 +40,13 @@ import com.google.api.client.googleapis.media.MediaHttpUploaderProgressListener;
 
 public class BigqueryWriter
 {
-
     private final Logger log = Exec.getLogger(BigqueryWriter.class);
-    private final String project;
-    private final String dataset;
-    private final String table;
     private final boolean autoCreateTable;
     private final Optional<String> schemaPath;
     private final TableSchema tableSchema;
     private final String sourceFormat;
     private final String fieldDelimiter;
-    private final int maxBadrecords;
+    private final int maxBadRecords;
     private final String encoding;
     private final boolean preventDuplicateInsert;
     private final long jobStatusMaxPollingTime;
@@ -60,16 +56,14 @@ public class BigqueryWriter
     private final boolean allowQuotedNewlines;
     private final Bigquery bigQueryClient;
 
-    public BigqueryWriter(Builder builder) throws FileNotFoundException, IOException, GeneralSecurityException
+    public BigqueryWriter(Builder builder)
+            throws IOException, GeneralSecurityException
     {
-        this.project = builder.project;
-        this.dataset = builder.dataset;
-        this.table = builder.table;
         this.autoCreateTable = builder.autoCreateTable;
         this.schemaPath = builder.schemaPath;
         this.sourceFormat = builder.sourceFormat.toUpperCase();
         this.fieldDelimiter = builder.fieldDelimiter;
-        this.maxBadrecords = builder.maxBadrecords;
+        this.maxBadRecords = builder.maxBadRecords;
         this.encoding = builder.encoding.toUpperCase();
         this.preventDuplicateInsert = builder.preventDuplicateInsert;
         this.jobStatusMaxPollingTime = builder.jobStatusMaxPollingTime;
@@ -81,15 +75,14 @@ public class BigqueryWriter
         BigqueryAuthentication auth = new BigqueryAuthentication(builder.authMethod, builder.serviceAccountEmail, builder.p12KeyFilePath, builder.applicationName);
         this.bigQueryClient = auth.getBigqueryClient();
 
-        checkConfig();
         if (autoCreateTable) {
-            this.tableSchema = createTableSchema(builder.schemaPath);
+            this.tableSchema = createTableSchema();
         } else {
             this.tableSchema = null;
         }
     }
 
-    private String getJobStatus(JobReference jobRef) throws JobFailedException
+    private String getJobStatus(String project, JobReference jobRef) throws JobFailedException
     {
         try {
             Job job = bigQueryClient.jobs().get(project, jobRef.getJobId()).execute();
@@ -108,7 +101,6 @@ public class BigqueryWriter
             String jobStatus = job.getStatus().getState();
             if (jobStatus.equals("DONE")) {
                 JobStatistics statistics = job.getStatistics();
-                //log.info(String.format("Job end. create:[%s] end:[%s]", statistics.getCreationTime(), statistics.getEndTime()));
                 log.info(String.format("Job statistics [%s]", statistics.getLoad()));
             }
             return jobStatus;
@@ -118,14 +110,14 @@ public class BigqueryWriter
         }
     }
 
-    private void getJobStatusUntilDone(JobReference jobRef) throws TimeoutException, JobFailedException
+    private void getJobStatusUntilDone(String project, JobReference jobRef) throws TimeoutException, JobFailedException
     {
         long startTime = System.currentTimeMillis();
         long elapsedTime;
 
         try {
             while (true) {
-                String jobStatus = getJobStatus(jobRef);
+                String jobStatus = getJobStatus(project, jobRef);
                 elapsedTime = System.currentTimeMillis() - startTime;
                 if (jobStatus.equals("DONE")) {
                     log.info(String.format("Job completed successfully. job id:[%s] elapsed_time:%dms status:[%s]", jobRef.getJobId(), elapsedTime, "SUCCESS"));
@@ -142,44 +134,28 @@ public class BigqueryWriter
         }
     }
 
-    public void executeLoad(String localFilePath) throws GoogleJsonResponseException, NoSuchAlgorithmException,
-            TimeoutException, JobFailedException, IOException
+    public void executeLoad(String project, String dataset, String table, String localFilePath)
+            throws NoSuchAlgorithmException, TimeoutException, JobFailedException, IOException
     {
         log.info(String.format("Job preparing... project:%s dataset:%s table:%s", project, dataset, table));
 
         Job job = new Job();
         JobReference jobRef = new JobReference();
-        JobConfiguration jobConfig = new JobConfiguration();
-        JobConfigurationLoad loadConfig = new JobConfigurationLoad();
-        jobConfig.setLoad(loadConfig);
+        JobConfiguration jobConfig = new JobConfiguration().setLoad(setLoadConfig(project, dataset, table));
         job.setConfiguration(jobConfig);
 
         if (preventDuplicateInsert) {
-            String jobId = createJobId(localFilePath);
+            ImmutableList<String> elements = ImmutableList.of(
+                    getLocalMd5hash(localFilePath), dataset, table,
+                    String.valueOf(tableSchema), sourceFormat, fieldDelimiter, String.valueOf(maxBadRecords),
+                    encoding, String.valueOf(ignoreUnknownValues), String.valueOf(allowQuotedNewlines)
+            );
+            String jobId = createJobId(elements);
+
             jobRef.setJobId(jobId);
             job.setJobReference(jobRef);
         }
 
-        loadConfig.setAllowQuotedNewlines(allowQuotedNewlines);
-        loadConfig.setEncoding(encoding);
-        loadConfig.setMaxBadRecords(maxBadrecords);
-        if (sourceFormat.equals("NEWLINE_DELIMITED_JSON")) {
-            loadConfig.setSourceFormat("NEWLINE_DELIMITED_JSON");
-        } else {
-            loadConfig.setFieldDelimiter(fieldDelimiter);
-        }
-        loadConfig.setWriteDisposition("WRITE_APPEND");
-        if (autoCreateTable) {
-            loadConfig.setSchema(tableSchema);
-            loadConfig.setCreateDisposition("CREATE_IF_NEEDED");
-            log.info(String.format("table:[%s] will be create if not exists", table));
-        } else {
-            loadConfig.setCreateDisposition("CREATE_NEVER");
-        }
-        loadConfig.setIgnoreUnknownValues(ignoreUnknownValues);
-
-        loadConfig.setDestinationTable(createTableReference());
-
         File file = new File(localFilePath);
         InputStreamContent mediaContent = new InputStreamContent("application/octet-stream",
                 new BufferedInputStream(
@@ -206,31 +182,52 @@ public class BigqueryWriter
         if (isSkipJobResultCheck) {
             log.info(String.format("Skip job status check. job id:[%s]", jobRef.getJobId()));
         } else {
-            getJobStatusUntilDone(jobRef);
+            getJobStatusUntilDone(project, jobRef);
         }
     }
 
-    private String createJobId(String localFilePath) throws NoSuchAlgorithmException, IOException
+    private JobConfigurationLoad setLoadConfig(String project, String dataset, String table)
+    {
+        JobConfigurationLoad config = new JobConfigurationLoad();
+        config.setAllowQuotedNewlines(allowQuotedNewlines)
+                .setEncoding(encoding)
+                .setMaxBadRecords(maxBadRecords)
+                .setSourceFormat(sourceFormat)
+                .setIgnoreUnknownValues(ignoreUnknownValues)
+                .setDestinationTable(createTableReference(project, dataset, table))
+                .setWriteDisposition("WRITE_APPEND");
+
+        if (sourceFormat.equals("CSV")) {
+            config.setFieldDelimiter(String.valueOf(fieldDelimiter));
+        }
+        if (autoCreateTable) {
+            config.setSchema(tableSchema);
+            config.setCreateDisposition("CREATE_IF_NEEDED");
+            log.info(String.format("table:[%s] will be create if not exists", table));
+        } else {
+            config.setCreateDisposition("CREATE_NEVER");
+        }
+        return config;
+    }
+
+    private String createJobId(ImmutableList<String> elements) throws NoSuchAlgorithmException, IOException
     {
         StringBuilder sb = new StringBuilder();
-        sb.append(getLocalMd5hash(localFilePath));
-        sb.append(dataset);
-        sb.append(table);
-        sb.append(tableSchema);
-        sb.append(sourceFormat);
-        sb.append(fieldDelimiter);
-        sb.append(maxBadrecords);
-        sb.append(encoding);
-        sb.append(ignoreUnknownValues);
+        for (String element : elements) {
+            sb.append(element);
+        }
 
         MessageDigest md = MessageDigest.getInstance("MD5");
-        String str = new String(sb);
-        byte[] digest = md.digest(str.getBytes());
+        byte[] digest = md.digest(new String(sb).getBytes());
         String hash = new String(Hex.encodeHex(digest));
-        return "embulk_job_" + hash;
+
+        StringBuilder jobId = new StringBuilder();
+        jobId.append("embulk_job_");
+        jobId.append(hash);
+        return jobId.toString();
     }
 
-    private TableReference createTableReference()
+    private TableReference createTableReference(String project, String dataset, String table)
     {
         return new TableReference()
                 .setProjectId(project)
@@ -238,7 +235,7 @@ public class BigqueryWriter
                 .setTableId(table);
     }
 
-    private TableSchema createTableSchema(Optional<String> schemaPath) throws FileNotFoundException, IOException
+    public TableSchema createTableSchema() throws IOException
     {
         String path = schemaPath.orNull();
         File file = new File(path);
@@ -247,8 +244,7 @@ public class BigqueryWriter
             stream = new FileInputStream(file);
             ObjectMapper mapper = new ObjectMapper();
             List<TableFieldSchema> fields = mapper.readValue(stream, new TypeReference<List<TableFieldSchema>>() {});
-            TableSchema tableSchema = new TableSchema().setFields(fields);
-            return tableSchema;
+            return new TableSchema().setFields(fields);
         } finally {
             if (stream != null) {
                 stream.close();
@@ -256,22 +252,22 @@ public class BigqueryWriter
         }
     }
 
-    public boolean isExistTable(String tableName) throws IOException
+    public boolean isExistTable(String project, String dataset, String table) throws IOException
     {
         Tables tableRequest = bigQueryClient.tables();
         try {
-            Table tableData = tableRequest.get(project, dataset, tableName).execute();
+            Table tableData = tableRequest.get(project, dataset, table).execute();
         } catch (GoogleJsonResponseException ex) {
             return false;
         }
         return true;
     }
 
-    public void checkConfig() throws FileNotFoundException, IOException
+    public void checkConfig(String project, String dataset, String table) throws IOException
     {
         if (autoCreateTable) {
             if (!schemaPath.isPresent()) {
-                throw new FileNotFoundException("schema_path is empty");
+                throw new FileNotFoundException("schema_file is empty");
             } else {
                 File file = new File(schemaPath.orNull());
                 if (!file.exists()) {
@@ -279,7 +275,7 @@ public class BigqueryWriter
                 }
             }
         } else {
-            if (!isExistTable(table)) {
+            if (!isExistTable(project, dataset, table)) {
                 throw new IOException(String.format("table [%s] is not exists", table));
             }
         }
@@ -341,14 +337,11 @@ public class BigqueryWriter
         private Optional<String> serviceAccountEmail;
         private Optional<String> p12KeyFilePath;
         private String applicationName;
-        private String project;
-        private String dataset;
-        private String table;
         private boolean autoCreateTable;
         private Optional<String> schemaPath;
         private String sourceFormat;
         private String fieldDelimiter;
-        private int maxBadrecords;
+        private int maxBadRecords;
         private String encoding;
         private boolean preventDuplicateInsert;
         private int jobStatusMaxPollingTime;
@@ -357,45 +350,12 @@ public class BigqueryWriter
         private boolean ignoreUnknownValues;
         private boolean allowQuotedNewlines;
 
-        public Builder(String authMethod)
+        public Builder(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath, String applicationName)
         {
             this.authMethod = authMethod;
-        }
-
-        public Builder setServiceAccountEmail(Optional<String> serviceAccountEmail)
-        {
             this.serviceAccountEmail = serviceAccountEmail;
-            return this;
-        }
-
-        public Builder setP12KeyFilePath(Optional<String> p12KeyFilePath)
-        {
             this.p12KeyFilePath = p12KeyFilePath;
-            return this;
-        }
-
-        public Builder setApplicationName(String applicationName)
-        {
             this.applicationName = applicationName;
-            return this;
-        }
-
-        public Builder setProject(String project)
-        {
-            this.project = project;
-            return this;
-        }
-
-        public Builder setDataset(String dataset)
-        {
-            this.dataset = dataset;
-            return this;
-        }
-
-        public Builder setTable(String table)
-        {
-            this.table = table;
-            return this;
         }
 
         public Builder setAutoCreateTable(boolean autoCreateTable)
@@ -422,9 +382,9 @@ public class BigqueryWriter
             return this;
         }
 
-        public Builder setMaxBadrecords(int maxBadrecords)
+        public Builder setMaxBadRecords(int maxBadRecords)
         {
-            this.maxBadrecords = maxBadrecords;
+            this.maxBadRecords = maxBadRecords;
             return this;
         }
 
@@ -482,4 +442,4 @@ public class BigqueryWriter
             super(message);
         }
     }
-}
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-20 00:00:00.000000000 Z
+date: 2015-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,7 +62,7 @@ files:
 - src/test/java/org/embulk/output/TestBigqueryWriter.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-output-bigquery-0.1.7.jar
+- classpath/embulk-output-bigquery-0.1.8.jar
 - classpath/google-api-client-1.20.0.jar
 - classpath/google-api-services-bigquery-v2-rev205-1.20.0.jar
 - classpath/google-http-client-1.20.0.jar
@@ -72,7 +72,7 @@ files:
 - classpath/httpclient-4.0.1.jar
 - classpath/httpcore-4.0.1.jar
 - classpath/jsr305-1.3.9.jar
-homepage: https://github.com/sakama/embulk-output-bigquery
+homepage: https://github.com/embulk/embulk-output-bigquery
 licenses:
 - Apache-2.0
 metadata: {}