embulk-output-td 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7b42edcec64619a1d24ce6b36967527ab76b4a63
-  data.tar.gz: 402e46fa95353e30cc38668c007893612cc2323c
+  metadata.gz: 7f75918c0833bfac5a887d29736ec84c572e9594
+  data.tar.gz: b4079b5ed3740e5f36c3f3c60763549dd2921ff8
 SHA512:
-  metadata.gz: 013f4c79c83f0d8215fb9d92098dbc75bb18f41b911117b82ae9773b010acba29a4124988fa1ca806abc57934840823059ad8c1ac13a439c40db9f1beea798b8
-  data.tar.gz: ba44f693d010a4967243cf9096019627fadf25d4728c016e9d5727196accb5fd70735225db2b35cee9897aa5440d414603921ead9d07400db4335fb909af74b2
+  metadata.gz: 80584f78e0fc8c85255e4a10bc53e2e39819253634f22d109e0c5542da4c4e83f7eb43fefc94f9d21ac8cdfe5be6ae3704684f6622b5728f5d6a66c4a9975602
+  data.tar.gz: 5134cd1a02a2c96c8330206d5ced8780ebae83954b825d8bf606b76cf44b9a31bb1801a9133ac5cc4b56c7a5305c66093f1c01839ee36d1113111edf8b2b4d0a

data/CHANGELOG.md

@@ -1,4 +1,12 @@
-## 0.1.6 - 2015-XX-XX
+## 0.1.7 - 2016-01-07
+
+* [new feature] Add time_value option  [#16](https://github.com/treasure-data/embulk-output-td/pull/16)
+* [new feature] Merge exact column types into the table schema [#25](https://github.com/treasure-data/embulk-output-td/pull/25)
+* [new feature]	Add stop_on_invalid_record option [#26](https://github.com/treasure-data/embulk-output-td/pull/26)
+* [new feature] Show skipped records by a perform job [#28](https://github.com/treasure-data/embulk-output-td/pull/28)
+* [maintenance] Use rename endpoint with 'overwrite' parameter [#23](https://github.com/treasure-data/embulk-output-td/pull/23)
+
+## 0.1.6 - 2015-12-08
 
 * [maintenance] Upgrade Embulk v0.7.10 [#22](https://github.com/treasure-data/embulk-output-td/pull/22)
 * [maintenance] Upgrade Embulk v0.7.5 [#21](https://github.com/treasure-data/embulk-output-td/pull/21)

data/README.md

@@ -24,6 +24,8 @@ TODO: Write short description here
 - **tmpdir**: temporal directory
 - **upload_concurrency**: upload concurrency (int, default=2). max concurrency is 8.
 - **file_split_size**: split size (long, default=16384 (16MB)).
+- **stop_on_invalid_record**: stop bulk load transaction if a file includes invalid record (such as invalid timestamp) (boolean, default=false).
+- **displayed_error_records_count_limit**: limit the count of the shown error records skipped by the perform job (int, default=10).
 - **default_timestamp_type_convert_to**: configure output type of timestamp columns. Available options are "sec" (convert timestamp to UNIX timestamp in seconds) and "string" (convert timestamp to string). (string, default: `"string"`)
 - **default_timezone**: default timezone (string, default='UTC')
 - **default_timestamp_format**: default timestamp format (string, default=`%Y-%m-%d %H:%M:%S.%6N`)
@@ -43,6 +45,12 @@ out:
   time_column: created_at
 ```
 
+## Install
+
+```
+$ embulk gem install embulk-output-td
+```
+
 ### Http Proxy Configuration
 If you want to add your Http Proxy configuration, you can use `http_proxy` parameter:
 ```yaml

data/build.gradle

@@ -16,7 +16,7 @@ configurations {
     provided
 }
 
-version = "0.1.6"
+version = "0.1.7"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7

data/embulk-output-td.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-td"
-  spec.version       = "0.1.6"
+  spec.version       = "0.1.7"
   spec.authors       = ["Muga Nishizawa"]
   spec.summary       = %[TreasureData output plugin for Embulk]
   spec.description   = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]

data/src/main/java/com/treasuredata/api/TdApiClient.java

@@ -3,8 +3,10 @@ package com.treasuredata.api;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableMap;
 import com.treasuredata.api.model.TDBulkImportSession;
+import com.treasuredata.api.model.TDColumn;
 import com.treasuredata.api.model.TDDatabase;
 import com.treasuredata.api.model.TDDatabaseList;
 import com.treasuredata.api.model.TDTable;
@@ -16,6 +18,8 @@ import org.eclipse.jetty.client.Origin;
 import org.eclipse.jetty.client.ProxyConfiguration;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.util.InputStreamContentProvider;
+import org.eclipse.jetty.client.util.InputStreamResponseListener;
 import org.eclipse.jetty.client.util.StringContentProvider;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.util.HttpCookieStore;
@@ -27,6 +31,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
 import java.security.MessageDigest;
@@ -37,6 +42,9 @@ import java.util.Date;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 public class TdApiClient
         implements Closeable
@@ -157,10 +165,21 @@ public class TdApiClient
         ContentResponse response = executeExchange(request);
     }
 
-    public void renameTable(String databaseName, String oldName, String newName)
+    public void renameTable(String databaseName, String oldName, String newName, boolean overwrite)
     {
         Request request = prepareExchange(HttpMethod.POST,
-                buildUrl("/v3/table/rename", databaseName, oldName, newName));
+                buildUrl("/v3/table/rename", databaseName, oldName, newName),
+                ImmutableMap.<String, String>of(),
+                ImmutableMap.of("overwrite", Boolean.toString(overwrite)));
+        ContentResponse response = executeExchange(request);
+    }
+
+    public void updateSchema(String databaseName, String tableName, List<TDColumn> newSchema)
+    {
+        Request request = prepareExchange(HttpMethod.POST,
+                buildUrl("/v3/table/update-schema", databaseName, tableName),
+                ImmutableMap.<String, String>of(),
+                ImmutableMap.of("schema", formatRequestParameterObject(newSchema)));
         ContentResponse response = executeExchange(request);
     }
 
@@ -231,6 +250,23 @@ public class TdApiClient
         ContentResponse response = executeExchange(request);
     }
 
+    public InputStream getBulkImportErrorRecords(String sessionName)
+    {
+        // TODO use td-client-java v0.7
+
+        Request request = prepareExchange(HttpMethod.GET,
+                buildUrl("/v3/bulk_import/error_records", sessionName));
+        InputStreamResponseListener listener = new InputStreamResponseListener();
+        request.send(listener);
+        try {
+            listener.get(60000, TimeUnit.MILLISECONDS); // 60 sec.
+            return listener.getInputStream();
+        }
+        catch (InterruptedException | ExecutionException | TimeoutException e) {
+            throw Throwables.propagate(e);
+        }
+    }
+
     private Request prepareExchange(HttpMethod method, String url)
     {
         return prepareExchange(method, url, Collections.<String, String>emptyMap(),

data/src/main/java/com/treasuredata/api/model/TDColumn.java

@@ -4,6 +4,14 @@ import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
 import com.fasterxml.jackson.databind.RuntimeJsonMappingException;
 import com.google.common.base.Objects;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import org.json.simple.JSONArray;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+import java.util.ArrayList;
+import java.util.List;
 
 public class TDColumn
 {
@@ -33,8 +41,44 @@ public class TDColumn
         return key;
     }
 
+    private static JSONArray castToArray(Object obj)
+    {
+        if (obj instanceof JSONArray) {
+            return (JSONArray) obj;
+        }
+        else {
+            throw new RuntimeJsonMappingException("Not an json array: " + obj);
+        }
+    }
+
+    public static List<TDColumn> parseTuple(String jsonStr)
+    {
+        if (Strings.isNullOrEmpty(jsonStr)) {
+            return new ArrayList<>(0);
+        }
+
+        // unescape json quotation
+        try {
+            String unescaped = jsonStr.replaceAll("\\\"", "\"");
+            JSONArray arr = castToArray(new JSONParser().parse(unescaped));
+            List<TDColumn> columnList = new ArrayList<>(arr.size());
+            for (Object e : arr) {
+                JSONArray columnNameAndType = castToArray(e);
+                String[] s = new String[columnNameAndType.size()];
+                for (int i = 0; i < columnNameAndType.size(); ++i) {
+                    s[i] = columnNameAndType.get(i).toString();
+                }
+                columnList.add(parseTuple(s));
+            }
+            return columnList;
+        }
+        catch (ParseException e) {
+            return new ArrayList<>(0);
+        }
+    }
+
     @JsonCreator
-    public static TDColumn valueFromTuple(String[] tuple)
+    public static TDColumn parseTuple(String[] tuple)
     {
         // TODO encode key in some ways
         if (tuple != null && tuple.length == 2) {

data/src/main/java/com/treasuredata/api/model/TDTable.java

@@ -16,11 +16,11 @@ public class TDTable
     public TDTable(
             @JsonProperty("name") String name,
             @JsonProperty("type") TDTableType type,
-            @JsonProperty("columns") List<TDColumn> columns)
+            @JsonProperty("schema") String schema)
     {
         this.name = name;
         this.type = type;
-        this.columns = columns;
+        this.columns = TDColumn.parseTuple(schema);
     }
 
     @JsonProperty

data/src/main/java/org/embulk/output/td/RecordWriter.java

@@ -5,11 +5,7 @@ import com.google.common.base.Stopwatch;
 import com.google.common.base.Throwables;
 import com.treasuredata.api.TdApiClient;
 import org.embulk.config.TaskReport;
-import org.embulk.output.td.writer.FieldWriter;
-import org.embulk.output.td.writer.IFieldWriter;
 import org.embulk.output.td.writer.FieldWriterSet;
-import org.embulk.spi.Column;
-import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.Exec;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageReader;
@@ -68,7 +64,8 @@ public class RecordWriter
         new FieldWriterSet(log, task, schema);
     }
 
-    void open(final Schema schema)
+    @VisibleForTesting
+    public void open(final Schema schema)
             throws IOException
     {
         this.pageReader = new PageReader(checkNotNull(schema));
@@ -84,7 +81,7 @@ public class RecordWriter
     }
 
     @VisibleForTesting
-    MsgpackGZFileBuilder getBuilder()
+    public MsgpackGZFileBuilder getBuilder()
     {
         return builder;
     }
@@ -96,52 +93,7 @@ public class RecordWriter
 
         try {
             while (pageReader.nextRecord()) {
-                builder.writeMapBegin(fieldWriters.getFieldCount());
-
-                pageReader.getSchema().visitColumns(new ColumnVisitor() {
-                    @Override
-                    public void booleanColumn(Column column)
-                    {
-                        write(column);
-                    }
-
-                    @Override
-                    public void longColumn(Column column)
-                    {
-                        write(column);
-                    }
-
-                    @Override
-                    public void doubleColumn(Column column)
-                    {
-                        write(column);
-                    }
-
-                    @Override
-                    public void stringColumn(Column column)
-                    {
-                        write(column);
-                    }
-
-                    @Override
-                    public void timestampColumn(Column column)
-                    {
-                        write(column);
-                    }
-
-                    private void write(Column column)
-                    {
-                        IFieldWriter fieldWriter = fieldWriters.getFieldWriter(column.getIndex());
-                        try {
-                            fieldWriter.writeKeyValue(builder, pageReader, column);
-                        }
-                        catch (IOException e) {
-                            throw Throwables.propagate(e);
-                        }
-                    }
-                });
-
-                builder.writeMapEnd();
+                fieldWriters.addRecord(builder, pageReader);
 
                 if (builder.getWrittenSize() > fileSplitSize) {
                     flush();

data/src/main/java/org/embulk/output/td/TdOutputPlugin.java

@@ -1,8 +1,15 @@
 package org.embulk.output.td;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.List;
+import java.util.ArrayList;
 import java.util.Map;
+import java.util.HashMap;
+import java.nio.charset.StandardCharsets;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+
 import javax.validation.constraints.Min;
 import javax.validation.constraints.Max;
 
@@ -19,6 +26,9 @@ import com.treasuredata.api.TdApiNotFoundException;
 import com.treasuredata.api.model.TDBulkImportSession;
 import com.treasuredata.api.model.TDBulkImportSession.ImportStatus;
 import com.treasuredata.api.model.TDTable;
+import com.treasuredata.api.model.TDColumn;
+import com.treasuredata.api.model.TDColumnType;
+import com.treasuredata.api.model.TDPrimitiveColumnType;
 import org.embulk.config.TaskReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -28,14 +38,20 @@ import org.embulk.config.ConfigException;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.output.td.writer.FieldWriterSet;
+import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
+import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.ExecSession;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Schema;
+import org.embulk.spi.Column;
 import org.embulk.spi.TransactionalPageOutput;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
 import org.joda.time.format.DateTimeFormat;
+import org.msgpack.MessagePack;
+import org.msgpack.unpacker.Unpacker;
+import org.msgpack.unpacker.UnpackerIterator;
 import org.slf4j.Logger;
 
 public class TdOutputPlugin
@@ -90,6 +106,10 @@ public class TdOutputPlugin
         @ConfigDefault("null")
         public Optional<String> getTimeColumn();
 
+        @Config("time_value")
+        @ConfigDefault("null")
+        public Optional<TimeValueConfig> getTimeValue(); // TODO allow timestamp format such as {from: "2015-01-01 00:00:00 UTC", to: "2015-01-02 00:00:00 UTC"} as well as unixtime integer
+
         @Config("unix_timestamp_unit")
         @ConfigDefault("\"sec\"")
         public UnixTimestampUnit getUnixTimestampUnit();
@@ -128,6 +148,15 @@ public class TdOutputPlugin
         @ConfigDefault("{}")
         public Map<String, TimestampColumnOption> getColumnOptions();
 
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
+
+        @Config("displayed_error_records_count_limit")
+        @ConfigDefault("10")
+        @Min(0)
+        int getDisplayedErrorRecordsCountLimit();
+
         public boolean getDoUpload();
         public void setDoUpload(boolean doUpload);
 
@@ -184,6 +213,19 @@ public class TdOutputPlugin
         public boolean getUseSsl();
     }
 
+    public interface TimeValueConfig
+            extends Task
+    {
+        @Config("from")
+        @Min(0)
+        public long getFrom();
+
+        @Config("to")
+        @ConfigDefault("0")
+        @Min(0)
+        public long getTo();
+    }
+
     public static enum ConvertTimestampType
     {
         STRING(-1),
@@ -314,7 +356,7 @@ public class TdOutputPlugin
             // validate FieldWriterSet configuration before transaction is started
             RecordWriter.validateSchema(log, task, schema);
 
-            return doRun(client, task, control);
+            return doRun(client, schema, task, control);
         }
     }
 
@@ -324,17 +366,17 @@ public class TdOutputPlugin
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
         try (TdApiClient client = newTdApiClient(task)) {
-            return doRun(client, task, control);
+            return doRun(client, schema, task, control);
         }
     }
 
     @VisibleForTesting
-    ConfigDiff doRun(TdApiClient client, PluginTask task, OutputPlugin.Control control)
+    ConfigDiff doRun(TdApiClient client, Schema schema, PluginTask task, OutputPlugin.Control control)
     {
         boolean doUpload = startBulkImportSession(client, task.getSessionName(), task.getDatabase(), task.getLoadTargetTableName());
         task.setDoUpload(doUpload);
         control.run(task.dump());
-        completeBulkImportSession(client, task.getSessionName(), 0);  // TODO perform job priority
+        completeBulkImportSession(client, schema, task, 0);  // TODO perform job priority
 
         // commit
         switch (task.getMode()) {
@@ -377,7 +419,7 @@ public class TdOutputPlugin
     }
 
     @VisibleForTesting
-    TdApiClient newTdApiClient(final PluginTask task)
+    public TdApiClient newTdApiClient(final PluginTask task)
     {
         Optional<HttpProxyConfig> httpProxyConfig = newHttpProxyConfig(task.getHttpProxy());
         TdApiClientConfig config = new TdApiClientConfig(task.getEndpoint(), task.getUseSsl(), httpProxyConfig);
@@ -519,8 +561,9 @@ public class TdOutputPlugin
     }
 
     @VisibleForTesting
-    void completeBulkImportSession(TdApiClient client, String sessionName, int priority)
+    void completeBulkImportSession(TdApiClient client, Schema schema, PluginTask task, int priority)
     {
+        String sessionName = task.getSessionName();
         TDBulkImportSession session = client.getBulkImportSession(sessionName);
 
         switch (session.getStatus()) {
@@ -548,12 +591,28 @@ public class TdOutputPlugin
             // pass
         case READY:
             // TODO add an option to make the transaction failed if error_records or error_parts is too large
-            // commit
+
+            // add Embulk's columns to the table schema
+            Map<String, TDColumnType> newColumns = updateSchema(client, schema, task);
             log.info("Committing bulk import session '{}'", sessionName);
             log.info("    valid records: {}", session.getValidRecords());
             log.info("    error records: {}", session.getErrorRecords());
             log.info("    valid parts: {}", session.getValidParts());
             log.info("    error parts: {}", session.getErrorParts());
+            if (!newColumns.isEmpty()) {
+                log.info("    new columns:");
+            }
+            for (Map.Entry<String, TDColumnType> pair : newColumns.entrySet()) {
+                log.info("      - {}: {}", pair.getKey(), pair.getValue());
+            }
+
+            showBulkImportErrorRecords(client, sessionName, (int) Math.min(session.getErrorRecords(), task.getDisplayedErrorRecordsCountLimit()));
+
+            if (session.getErrorRecords() > 0 && task.getStopOnInvalidRecord()) {
+                throw new DataException(String.format("Stop committing because the perform job skipped %d error records", session.getErrorRecords()));
+            }
+
+            // commit
             client.commitBulkImportSession(sessionName);
 
             // pass
@@ -571,6 +630,105 @@ public class TdOutputPlugin
         }
     }
 
+    Map<String, TDColumnType> updateSchema(TdApiClient client, Schema inputSchema, PluginTask task)
+    {
+        String databaseName = task.getDatabase();
+
+        TDTable table = findTable(client, databaseName, task.getTable());
+        if (table == null) {
+            return new HashMap<>();
+        }
+
+        final Map<String, TDColumnType> guessedSchema = new HashMap<>();
+        inputSchema.visitColumns(new ColumnVisitor() {
+            public void booleanColumn(Column column)
+            {
+                guessedSchema.put(column.getName(), TDPrimitiveColumnType.LONG);
+            }
+
+            public void longColumn(Column column)
+            {
+                guessedSchema.put(column.getName(), TDPrimitiveColumnType.LONG);;
+            }
+
+            public void doubleColumn(Column column)
+            {
+                guessedSchema.put(column.getName(), TDPrimitiveColumnType.DOUBLE);
+            }
+
+            public void stringColumn(Column column)
+            {
+                guessedSchema.put(column.getName(), TDPrimitiveColumnType.STRING);
+            }
+
+            public void timestampColumn(Column column)
+            {
+                guessedSchema.put(column.getName(), TDPrimitiveColumnType.STRING);
+            }
+        });
+
+        Map<String, Integer> usedNames = new HashMap<>();
+        for (TDColumn existent : table.getColumns()) {
+            usedNames.put(new String(existent.getKey()), 1);
+            guessedSchema.remove(existent.getName()); // don't change type of existent columns
+        }
+        guessedSchema.remove("time"); // don't change type of 'time' column
+
+        List<TDColumn> newSchema = new ArrayList<>(table.getColumns());
+        for (Map.Entry<String, TDColumnType> pair : guessedSchema.entrySet()) {
+            String key = renameColumn(pair.getKey());
+
+            if (!usedNames.containsKey(key)) {
+                usedNames.put(key, 1);
+            } else {
+                int next = usedNames.get(key);
+                key = key + "_" + next;
+                usedNames.put(key, next + 1);
+            }
+
+            newSchema.add(new TDColumn(pair.getKey(), pair.getValue(), key.getBytes(StandardCharsets.UTF_8)));
+        }
+
+        client.updateSchema(databaseName, task.getLoadTargetTableName(), newSchema);
+        return guessedSchema;
+    }
+
+    private static TDTable findTable(TdApiClient client, String databaseName, String tableName)
+    {
+        for (TDTable table : client.getTables(databaseName)) {
+            if (table.getName().equals(tableName)) {
+                return table;
+            }
+        }
+        return null;
+    }
+
+    private static final Pattern COLUMN_NAME_PATTERN = Pattern.compile("\\A[a-z_][a-z0-9_]*\\z");
+    private static final Pattern COLUMN_NAME_SQUASH_PATTERN = Pattern.compile("(?:[^a-zA-Z0-9_]|(?:\\A[^a-zA-Z_]))+");
+
+    private static String renameColumn(String origName)
+    {
+        if (COLUMN_NAME_PATTERN.matcher(origName).matches()) {
+            return origName;
+        }
+        return COLUMN_NAME_SQUASH_PATTERN.matcher(origName).replaceAll("_").toLowerCase();
+    }
+
+    void showBulkImportErrorRecords(TdApiClient client, String sessionName, int recordCountLimit)
+    {
+        log.info("Show {} error records", recordCountLimit);
+        try (InputStream in = client.getBulkImportErrorRecords(sessionName)) {
+            Unpacker unpacker = new MessagePack().createUnpacker(new GZIPInputStream(in));
+            UnpackerIterator records = unpacker.iterator();
+            for (int i = 0; i < recordCountLimit; i++) {
+                log.info("    {}", records.next());
+            }
+        }
+        catch (Exception ignored) {
+            log.info("Stop downloading error records", ignored);
+        }
+    }
+
     @VisibleForTesting
     TDBulkImportSession waitForStatusChange(TdApiClient client, String sessionName,
             ImportStatus current, ImportStatus expecting, String operation)
@@ -604,23 +762,7 @@ public class TdOutputPlugin
     void renameTable(TdApiClient client, String databaseName, String oldName, String newName)
     {
         log.debug("Renaming table \"{}\".\"{}\" to \"{}\"", databaseName, oldName, newName);
-        try {
-            client.renameTable(databaseName, oldName, newName);
-        }
-        catch (TdApiConflictException e) {
-            try {
-                client.deleteTable(databaseName, newName);
-                log.debug("Deleted original table \"{}\".\"{}\"", databaseName, newName);
-            }
-            catch (TdApiNotFoundException ex) {
-                // ignoreable error
-            }
-            catch (IOException ex) {
-                throw Throwables.propagate(ex);
-            }
-
-            client.renameTable(databaseName, oldName, newName);
-        }
+        client.renameTable(databaseName, oldName, newName, true);
     }
 
     @Override

data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java

@@ -1,9 +1,15 @@
 package org.embulk.output.td.writer;
 
+import java.io.IOException;
+
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
 import org.embulk.config.ConfigException;
 import org.embulk.output.td.TdOutputPlugin;
 import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.type.BooleanType;
@@ -13,6 +19,8 @@ import org.embulk.spi.type.StringType;
 import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.util.Timestamps;
+import org.embulk.output.td.MsgpackGZFileBuilder;
+import org.embulk.output.td.TdOutputPlugin.TimeValueConfig;
 import org.slf4j.Logger;
 
 public class FieldWriterSet
@@ -26,11 +34,17 @@ public class FieldWriterSet
 
     private final int fieldCount;
     private final IFieldWriter[] fieldWriters;
+    private final Optional<TimeValueGenerator> staticTimeValue;
 
     public FieldWriterSet(Logger log, TdOutputPlugin.PluginTask task, Schema schema)
     {
         Optional<String> userDefinedPrimaryKeySourceColumnName = task.getTimeColumn();
         TdOutputPlugin.ConvertTimestampType convertTimestamp = task.getConvertTimestampType();
+        Optional<TimeValueConfig> timeValueConfig = task.getTimeValue();
+        if (timeValueConfig.isPresent() && userDefinedPrimaryKeySourceColumnName.isPresent()) {
+            throw new ConfigException("Setting both time_column and time_value is invalid");
+        }
+
         boolean hasPkWriter = false;
         int duplicatePrimaryKeySourceIndex = -1;
         int firstTimestampColumnIndex = -1;
@@ -62,7 +76,13 @@ public class FieldWriterSet
                     columnName = newColumnUniqueName(columnName, schema);
                     mode = ColumnWriterMode.SIMPLE_VALUE;
                     log.warn("time_column '{}' is set but 'time' column also exists. The existent 'time' column is renamed to {}",
-                            userDefinedPrimaryKeySourceColumnName.get(), "time", "time", columnName);
+                            userDefinedPrimaryKeySourceColumnName.get(), columnName);
+                }
+                else if (timeValueConfig.isPresent()) {
+                    columnName = newColumnUniqueName(columnName, schema);
+                    mode = ColumnWriterMode.SIMPLE_VALUE;
+                    log.warn("time_value is set but 'time' column also exists. The existent 'time' column is renamed to {}",
+                            columnName);
                 }
                 else {
                     mode = ColumnWriterMode.PRIMARY_KEY;
@@ -143,7 +163,11 @@ public class FieldWriterSet
             fc += 1;
         }
 
-        if (!hasPkWriter) {
+        if (timeValueConfig.isPresent()) {
+            // "time" column is written by RecordWriter
+            fc += 1;
+        }
+        else if (!hasPkWriter) {
             // PRIMARY_KEY was not found.
             if (duplicatePrimaryKeySourceIndex < 0) {
                 if (userDefinedPrimaryKeySourceColumnName.isPresent()) {
@@ -195,6 +219,13 @@ public class FieldWriterSet
             fc += 1;
         }
 
+        if (timeValueConfig.isPresent()) {
+            staticTimeValue = Optional.of(new TimeValueGenerator(timeValueConfig.get()));
+        }
+        else {
+            staticTimeValue = Optional.absent();
+        }
+
         fieldCount = fc;
     }
 
@@ -218,13 +249,101 @@ public class FieldWriterSet
         return false;
     }
 
+    @VisibleForTesting
     public IFieldWriter getFieldWriter(int index)
     {
         return fieldWriters[index];
     }
 
-    public int getFieldCount()
+    public void addRecord(final MsgpackGZFileBuilder builder, final PageReader reader)
+            throws IOException
     {
-        return fieldCount;
+        beginRecord(builder);
+
+        reader.getSchema().visitColumns(new ColumnVisitor() {
+            @Override
+            public void booleanColumn(Column column)
+            {
+                addColumn(builder, reader, column);
+            }
+
+            @Override
+            public void longColumn(Column column)
+            {
+                addColumn(builder, reader, column);
+            }
+
+            @Override
+            public void doubleColumn(Column column)
+            {
+                addColumn(builder, reader, column);
+            }
+
+            @Override
+            public void stringColumn(Column column)
+            {
+                addColumn(builder, reader, column);
+            }
+
+            @Override
+            public void timestampColumn(Column column)
+            {
+                addColumn(builder, reader, column);
+            }
+
+        });
+
+        endRecord(builder);
+    }
+
+    private void beginRecord(MsgpackGZFileBuilder builder)
+            throws IOException
+    {
+        builder.writeMapBegin(fieldCount);
+        if (staticTimeValue.isPresent()) {
+            builder.writeString("time");
+            builder.writeLong(staticTimeValue.get().next());
+        }
+    }
+
+    private void endRecord(MsgpackGZFileBuilder builder)
+            throws IOException
+    {
+        builder.writeMapEnd();
+    }
+
+    private void addColumn(MsgpackGZFileBuilder builder, PageReader reader, Column column)
+    {
+        try {
+            fieldWriters[column.getIndex()].writeKeyValue(builder, reader, column);
+        }
+        catch (IOException e) {
+            throw Throwables.propagate(e);
+        }
+    }
+
+    static class TimeValueGenerator
+    {
+        private final long from;
+        private final long to;
+        private long current;
+
+        TimeValueGenerator(TimeValueConfig config)
+        {
+            current = from = config.getFrom();
+            to = config.getTo();
+        }
+
+        long next()
+        {
+            try {
+                return current++;
+            }
+            finally {
+                if (current >= to) {
+                    current = from;
+                }
+            }
+        }
     }
 }

data/src/test/java/org/embulk/output/td/TestRecordWriter.java

@@ -1,9 +1,9 @@
 package org.embulk.output.td;
 
+import com.google.common.collect.ImmutableMap;
 import com.treasuredata.api.TdApiClient;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.output.td.TdOutputPlugin.PluginTask;
-import org.embulk.output.td.writer.FieldWriterSet;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageTestUtils;
 import org.embulk.spi.Schema;
@@ -27,6 +27,7 @@ import static org.embulk.output.td.TestTdOutputPlugin.plugin;
 import static org.embulk.output.td.TestTdOutputPlugin.pluginTask;
 import static org.embulk.output.td.TestTdOutputPlugin.schema;
 import static org.embulk.output.td.TestTdOutputPlugin.recordWriter;
+import static org.embulk.output.td.TestTdOutputPlugin.tdApiClient;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
@@ -58,16 +59,14 @@ public class TestRecordWriter
 
         plugin = plugin();
         task = pluginTask(config().set("session_name", "my_session"));
-
-        TdApiClient client = plugin.newTdApiClient(task);
-        FieldWriterSet fieldWriters = fieldWriters(log, task, schema);
-        recordWriter = recordWriter(task, client, fieldWriters);
     }
 
     @Test
     public void checkOpenAndClose()
             throws Exception
     {
+        recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
+
         // confirm that no error happens
         try {
             recordWriter.open(schema);
@@ -82,6 +81,7 @@ public class TestRecordWriter
             throws Exception
     {
         TdApiClient client = spy(plugin.newTdApiClient(task));
+        recordWriter = recordWriter(task, client, fieldWriters(log, task, schema));
 
         { // add no record
             RecordWriter recordWriter = recordWriter(task, client, fieldWriters(log, task, schema));
@@ -116,6 +116,8 @@ public class TestRecordWriter
     public void addNonNullValues()
             throws Exception
     {
+        recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
+
         try {
             recordWriter.open(schema);
 
@@ -152,6 +154,8 @@ public class TestRecordWriter
     public void addNullValues()
             throws Exception
     {
+        recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
+
         try {
             recordWriter.open(schema);
 
@@ -183,9 +187,51 @@ public class TestRecordWriter
         }
     }
 
+    @Test
+    public void checkGeneratedTimeValueByOption()
+            throws Exception
+    {
+        schema = schema("_c0", Types.LONG, "_c1", Types.STRING,
+                "_c2", Types.BOOLEAN, "_c3", Types.DOUBLE, "_c4", Types.TIMESTAMP);
+        task = pluginTask(config().set("session_name", "my_session").set("time_value", ImmutableMap.of("from", 0L, "to", 0L)));
+        recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
+
+        try {
+            recordWriter.open(schema);
+
+            // values are not null
+            for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema,
+                    0L, "v", true, 0.0, Timestamp.ofEpochSecond(1442595600L))) {
+                recordWriter.add(page);
+            }
+
+            MsgpackGZFileBuilder builder = recordWriter.getBuilder();
+            builder.finish();
+
+            // record count 1
+            assertEquals(1, builder.getRecordCount());
+
+            Unpacker u = msgpack.createUnpacker(new GZIPInputStream(new FileInputStream(builder.getFile())));
+            MapValue v = u.readValue().asMapValue();
+
+            // compare actual values
+            assertEquals(0L, v.get(createRawValue("time")).asIntegerValue().getLong());
+            assertEquals(0L, v.get(createRawValue("_c0")).asIntegerValue().getLong());
+            assertEquals("v", v.get(createRawValue("_c1")).asRawValue().getString());
+            assertEquals(true, v.get(createRawValue("_c2")).asBooleanValue().getBoolean());
+            assertEquals(0.0, v.get(createRawValue("_c3")).asFloatValue().getDouble(), 0.000001);
+            assertEquals("2015-09-18 17:00:00.000", v.get(createRawValue("_c4")).asRawValue().getString());
+
+        }
+        finally {
+            recordWriter.close();
+        }
+    }
+
     @Test
     public void doAbortNorthing()
     {
+        recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
         recordWriter.abort();
         // no error happen
     }
@@ -193,6 +239,7 @@ public class TestRecordWriter
     @Test
     public void checkTaskReport()
     {
+        recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
         assertTrue(recordWriter.commit().isEmpty());
     }
 }

data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java

@@ -8,6 +8,7 @@ import com.treasuredata.api.TdApiConflictException;
 import com.treasuredata.api.TdApiNotFoundException;
 import com.treasuredata.api.model.TDBulkImportSession;
 import com.treasuredata.api.model.TDBulkImportSession.ImportStatus;
+import com.treasuredata.api.model.TDColumnType;
 import com.treasuredata.api.model.TDTable;
 import com.treasuredata.api.model.TDTableType;
 import org.embulk.EmbulkTestRuntime;
@@ -34,6 +35,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.slf4j.Logger;
 
+import java.util.HashMap;
 import java.util.List;
 
 import static com.treasuredata.api.model.TDBulkImportSession.ImportStatus.COMMITTED;
@@ -112,7 +114,7 @@ public class TestTdOutputPlugin
     {
         doReturn("session_name").when(plugin).buildBulkImportSessionName(any(PluginTask.class), any(ExecSession.class));
         ConfigDiff configDiff = Exec.newConfigDiff().set("last_session", "session_name");
-        doReturn(configDiff).when(plugin).doRun(any(TdApiClient.class), any(PluginTask.class), any(OutputPlugin.Control.class));
+        doReturn(configDiff).when(plugin).doRun(any(TdApiClient.class), any(Schema.class), any(PluginTask.class), any(OutputPlugin.Control.class));
         Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
 
         { // auto_create_table is true
@@ -151,7 +153,7 @@ public class TestTdOutputPlugin
         task.setLoadTargetTableName("my_table");
         task.setDoUpload(true);
         doReturn(true).when(plugin).startBulkImportSession(any(TdApiClient.class), anyString(), anyString(), anyString());
-        doNothing().when(plugin).completeBulkImportSession(any(TdApiClient.class), anyString(), anyInt());
+        doNothing().when(plugin).completeBulkImportSession(any(TdApiClient.class), any(Schema.class), any(PluginTask.class), anyInt());
         Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
 
         ConfigDiff configDiff = plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
@@ -384,7 +386,10 @@ public class TestTdOutputPlugin
     public void completeBulkImportSession()
     {
         PluginTask task = pluginTask(config);
+        Schema schema = schema("c0", Types.LONG);
+
         doReturn(session(UNKNOWN, false)).when(plugin).waitForStatusChange(any(TdApiClient.class), anyString(), any(ImportStatus.class), any(ImportStatus.class), anyString());
+        doReturn(new HashMap<String, TDColumnType>()).when(plugin).updateSchema(any(TdApiClient.class), any(Schema.class), any(PluginTask.class));
 
         TdApiClient client = spy(plugin.newTdApiClient(task));
         doNothing().when(client).freezeBulkImportSession(anyString());
@@ -392,45 +397,45 @@ public class TestTdOutputPlugin
         doNothing().when(client).commitBulkImportSession(anyString());
 
         { // uploading + unfreeze
-            doReturn(session(UPLOADING, false)).when(client).getBulkImportSession("my_session");
-            plugin.completeBulkImportSession(client, "my_session", 0);
+            doReturn(session(UPLOADING, false)).when(client).getBulkImportSession(anyString());
+            plugin.completeBulkImportSession(client, schema, task, 0);
             // no error happens
         }
 
         { // uploading + frozen
-            doReturn(session(UPLOADING, true)).when(client).getBulkImportSession("my_session");
-            plugin.completeBulkImportSession(client, "my_session", 0);
+            doReturn(session(UPLOADING, true)).when(client).getBulkImportSession(anyString());
+            plugin.completeBulkImportSession(client, schema, task, 0);
             // no error happens
         }
 
         { // performing
             doReturn(session(PERFORMING, false)).when(client).getBulkImportSession(anyString());
-            plugin.completeBulkImportSession(client, "my_session", 0);
+            plugin.completeBulkImportSession(client, schema, task, 0);
             // no error happens
         }
 
         { // ready
             doReturn(session(READY, false)).when(client).getBulkImportSession(anyString());
-            plugin.completeBulkImportSession(client, "my_session", 0);
+            plugin.completeBulkImportSession(client, schema, task, 0);
             // no error happens
         }
 
         { // committing
             doReturn(session(COMMITTING, false)).when(client).getBulkImportSession(anyString());
-            plugin.completeBulkImportSession(client, "my_session", 0);
+            plugin.completeBulkImportSession(client, schema, task, 0);
             // no error happens
         }
 
         { // committed
             doReturn(session(COMMITTED, false)).when(client).getBulkImportSession(anyString());
-            plugin.completeBulkImportSession(client, "my_session", 0);
+            plugin.completeBulkImportSession(client, schema, task, 0);
             // no error happens
         }
 
         { // unknown
             doReturn(session(UNKNOWN, false)).when(client).getBulkImportSession(anyString());
             try {
-                plugin.completeBulkImportSession(client, "my_session", 0);
+                plugin.completeBulkImportSession(client, schema, task, 0);
                 fail();
             }
             catch (Throwable t) {
@@ -439,8 +444,8 @@ public class TestTdOutputPlugin
 
         { // if freezeBulkImportSession got 409, it can be ignoreable.
             doThrow(conflict()).when(client).freezeBulkImportSession(anyString());
-            doReturn(session(UPLOADING, true)).when(client).getBulkImportSession("my_session");
-            plugin.completeBulkImportSession(client, "my_session", 0);
+            doReturn(session(UPLOADING, true)).when(client).getBulkImportSession(anyString());
+            plugin.completeBulkImportSession(client, schema, task, 0);
             // no error happens
         }
     }
@@ -507,6 +512,11 @@ public class TestTdOutputPlugin
         return spy(new TdOutputPlugin());
     }
 
+    public static TdApiClient tdApiClient(TdOutputPlugin plugin, PluginTask task)
+    {
+        return spy(plugin.newTdApiClient(task));
+    }
+
     public static FieldWriterSet fieldWriters(Logger log, PluginTask task, Schema schema)
     {
         return spy(new FieldWriterSet(log, task, schema));

data/src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java

@@ -1,5 +1,6 @@
 package org.embulk.output.td.writer;
 
+import com.google.common.collect.ImmutableMap;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
@@ -79,6 +80,17 @@ public class TestFieldWriterSet
                 assertTrue(t instanceof ConfigException);
             }
         }
+
+        { // if both of time_column and time_value are specified, it throws ConfigError.
+            schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
+            try {
+                new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c1").set("time_value", ImmutableMap.of("from", 0L, "to", 0L))), schema);
+                fail();
+            }
+            catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
     }
 
     @Test
@@ -101,7 +113,7 @@ public class TestFieldWriterSet
     }
 
     @Test
-    public void specifiedTimeColumnOption()
+    public void specifiedTimeColumn()
     {
         { // time_column option (timestamp type)
             Schema schema = schema("_c0", Types.TIMESTAMP, "_c1", Types.STRING);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-td
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-08 00:00:00.000000000 Z
+date: 2016-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -103,7 +103,7 @@ files:
 - src/test/java/org/embulk/output/td/TestRecordWriter.java
 - src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
 - src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
-- classpath/embulk-output-td-0.1.6.jar
+- classpath/embulk-output-td-0.1.7.jar
 - classpath/javassist-3.18.1-GA.jar
 - classpath/jetty-client-9.2.2.v20140723.jar
 - classpath/jetty-http-9.2.2.v20140723.jar