embulk-output-td 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4fee7360589ff074152102355e21cd14a4fd6f15
-  data.tar.gz: f839023cacdc9b64382aa5d7b3fcdce51c997a86
+  metadata.gz: 5610036db652164098fcc3f828b8fb3f95542329
+  data.tar.gz: a297a28131574b748d20c49097774b6bdb607552
 SHA512:
-  metadata.gz: e841eb70257ada5e8c012595968536236f606aebe110fe2b32edf203607d569c5436d001504d0c5d98da490b58580893ea8b0fe58314fc8ae51c984cc0c7f0b8
-  data.tar.gz: 52af81b3e7c4d27ec844b5f74abe02a5dc2df4339945bd2da37580f5a184c5417ac1d2306fb22b62d164273df90b12e637e368eed2564d30d676443e2eea7b69
+  metadata.gz: 18bd27aedf3f1585c2c1fbbdc881f2e50157bfe61eb5d820363dd7896dc9c63f2c7fc2abae1ff1ef57c3b338b4672ee86bb1b57c097f073f0a1489a67800b3c4
+  data.tar.gz: 2d25527c5fb487970c90386c343c3dab2e4a0ad630698c9c772af247949f5affc5cae09d7edcd6ecdd43e21dc92d5c7a99055c4e9d5fc42d86b5643808867c0b

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.1.4 - 2015-XX-XX
+
+## 0.1.3 - 2015-08-05
+
+* [maintenance] Upgrade Embulk v0.6.19
+* [new feature] Add column_options [#11](https://github.com/treasure-data/embulk-output-td/pull/11)
+
 ## 0.1.2 - 2015-07-14
 
 ## 0.1.1 - 2015-07-14

data/README.md

@@ -23,6 +23,11 @@ TODO: Write short description here
 - **tmpdir**: temporal directory
 - **upload_concurrency**: upload concurrency (int, default=2). max concurrency is 8.
 - **file_split_size**: split size (long, default=16384 (16MB)).
+- **default_timezone**: default timezone (string, default='UTC')
+- **default_timestamp_format**: default timestamp format (string, default=`%Y-%m-%d %H:%M:%S.%6N`)
+- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
+  - **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
+  - **format**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, value of default_timestamp_format option is used by default)
 
 ## Example
 Here is sample configuration for TD output plugin.

data/build.gradle

@@ -13,19 +13,22 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.6.10"
-    provided "org.embulk:embulk-core:0.6.10"
-    compile  "org.eclipse.jetty:jetty-client:9.2.2.v20140723"
-    compile  "org.msgpack:msgpack:0.6.11"
+    compile     "org.embulk:embulk-core:0.6.19"
+    provided    "org.embulk:embulk-core:0.6.19"
+    compile     "org.embulk:embulk-standards:0.6.19"
+    provided    "org.embulk:embulk-standards:0.6.19"
+    compile     "org.eclipse.jetty:jetty-client:9.2.2.v20140723"
+    compile     "org.msgpack:msgpack:0.6.11"
 
     testCompile "junit:junit:4.+"
+    testCompile "org.bigtesting:fixd:1.0.0"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/embulk-output-td.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-td"
-  spec.version       = "0.1.2"
+  spec.version       = "0.1.3"
   spec.authors       = ["Muga Nishizawa"]
   spec.summary       = %[TreasureData output plugin for Embulk]
   spec.description   = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]

data/lib/embulk/output/td.rb

@@ -1,3 +1,3 @@
 Embulk::JavaPlugin.register_output(
-  "td", "org.embulk.output.TdOutputPlugin",
+  "td", "org.embulk.output.td.TdOutputPlugin",
   File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/output/{FinalizableExecutorService.java → td/FinalizableExecutorService.java}

@@ -1,4 +1,4 @@
-package org.embulk.output;
+package org.embulk.output.td;
 
 import java.io.Closeable;
 import java.io.IOException;

data/src/main/java/org/embulk/output/{MsgpackGZFileBuilder.java → td/MsgpackGZFileBuilder.java}

@@ -1,4 +1,4 @@
-package org.embulk.output;
+package org.embulk.output.td;
 
 import org.embulk.spi.Exec;
 import org.msgpack.MessagePack;

data/src/main/java/org/embulk/output/{RecordWriter.java → td/RecordWriter.java}

@@ -1,4 +1,4 @@
-package org.embulk.output;
+package org.embulk.output.td;
 
 import com.google.common.base.Optional;
 import com.google.common.base.Stopwatch;
@@ -6,7 +6,6 @@ import com.google.common.base.Throwables;
 import com.treasuredata.api.TdApiClient;
 import org.embulk.config.CommitReport;
 import org.embulk.config.ConfigException;
-import org.embulk.output.TdOutputPlugin.PluginTask;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.Exec;
@@ -21,6 +20,7 @@ import org.embulk.spi.type.LongType;
 import org.embulk.spi.type.StringType;
 import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.type.Type;
+import org.embulk.spi.util.Timestamps;
 import org.joda.time.DateTimeZone;
 import org.jruby.embed.ScriptingContainer;
 import org.msgpack.MessagePack;
@@ -35,7 +35,6 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 
 import static com.google.common.base.Preconditions.checkNotNull;
-import org.embulk.output.TdOutputPlugin.UnixTimestampUnit;
 
 public class RecordWriter
         implements TransactionalPageOutput
@@ -57,7 +56,7 @@ public class RecordWriter
     private final int uploadConcurrency;
     private final long fileSplitSize; // unit: kb
 
-    public RecordWriter(PluginTask task, int taskIndex, TdApiClient client, FieldWriterSet fieldWriters)
+    public RecordWriter(TdOutputPlugin.PluginTask task, int taskIndex, TdApiClient client, FieldWriterSet fieldWriters)
     {
         this.log = Exec.getLogger(getClass());
         this.client = checkNotNull(client);
@@ -72,7 +71,7 @@ public class RecordWriter
         this.fileSplitSize = task.getFileSplitSize() * 1024;
     }
 
-    public static void validateSchema(Logger log, PluginTask task, Schema schema)
+    public static void validateSchema(Logger log, TdOutputPlugin.PluginTask task, Schema schema)
     {
         new FieldWriterSet(log, task, schema);
     }
@@ -263,7 +262,7 @@ public class RecordWriter
         private final int fieldCount;
         private final FieldWriter[] fieldWriters;
 
-        public FieldWriterSet(Logger log, PluginTask task, Schema schema)
+        public FieldWriterSet(Logger log, TdOutputPlugin.PluginTask task, Schema schema)
         {
             Optional<String> userDefinedPrimaryKeySourceColumnName = task.getTimeColumn();
             boolean hasPkWriter = false;
@@ -272,6 +271,7 @@ public class RecordWriter
 
             int fc = 0;
             fieldWriters = new FieldWriter[schema.size()];
+            TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
 
             for (int i = 0; i < schema.size(); i++) {
                 String columnName = schema.getColumnName(i);
@@ -311,13 +311,13 @@ public class RecordWriter
                     case PRIMARY_KEY:
                         log.info("Using {}:{} column as the data partitioning key", columnName, columnType);
                         if (columnType instanceof LongType) {
-                            if (task.getUnixTimestampUnit() != UnixTimestampUnit.SEC) {
+                            if (task.getUnixTimestampUnit() != TdOutputPlugin.UnixTimestampUnit.SEC) {
                                 log.warn("time column is converted from {} to seconds", task.getUnixTimestampUnit());
                             }
                             writer = new UnixTimestampLongFieldWriter(columnName, task.getUnixTimestampUnit().getFractionUnit());
                             hasPkWriter = true;
                         } else if (columnType instanceof TimestampType) {
-                            writer = new TimestampStringFieldWriter(task.getJRuby(), columnName);
+                            writer = new TimestampStringFieldWriter(timestampFormatters[i], columnName);
                             hasPkWriter = true;
                         } else {
                             throw new ConfigException(String.format("Type of '%s' column must be long or timestamp but got %s",
@@ -335,7 +335,7 @@ public class RecordWriter
                         } else if (columnType instanceof StringType) {
                             writer = new StringFieldWriter(columnName);
                         } else if (columnType instanceof TimestampType) {
-                            writer = new TimestampStringFieldWriter(task.getJRuby(), columnName);
+                            writer = new TimestampStringFieldWriter(timestampFormatters[i], columnName);
                             if (firstTimestampColumnIndex < 0) {
                                 firstTimestampColumnIndex = i;
                             }
@@ -381,7 +381,7 @@ public class RecordWriter
                 } else if (columnType instanceof TimestampType) {
                     log.info("Duplicating {}:{} column to 'time' column as seconds for the data partitioning",
                             columnName, columnType);
-                    writer = new TimestampFieldLongDuplicator(task.getJRuby(), columnName, "time");
+                    writer = new TimestampFieldLongDuplicator(timestampFormatters[duplicatePrimaryKeySourceIndex], columnName, "time");
                 } else {
                     throw new ConfigException(String.format("Type of '%s' column must be long or timestamp but got %s",
                             columnName, columnType));
@@ -541,20 +541,19 @@ public class RecordWriter
     static class TimestampStringFieldWriter
             extends FieldWriter
     {
-        // to format timestamp values to string by "%Y-%m-%d %H:%M:%S.%3N"
-        private final TimestampFormatter defaultFormatter;
+        private final TimestampFormatter formatter;
 
-        public TimestampStringFieldWriter(ScriptingContainer jruby, String keyName)
+        public TimestampStringFieldWriter(TimestampFormatter formatter, String keyName)
         {
             super(keyName);
-            this.defaultFormatter = new TimestampFormatter(jruby, "%Y-%m-%d %H:%M:%S.%3N", DateTimeZone.UTC);
+            this.formatter = formatter;
         }
 
         @Override
         public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
                 throws IOException
         {
-            builder.writeString(defaultFormatter.format(reader.getTimestamp(column)));
+            builder.writeString(formatter.format(reader.getTimestamp(column)));
         }
     }
 
@@ -599,9 +598,9 @@ public class RecordWriter
     {
         private final TimestampLongFieldWriter timeFieldWriter;
 
-        public TimestampFieldLongDuplicator(ScriptingContainer jruby, String keyName, String longDuplicateKeyName)
+        public TimestampFieldLongDuplicator(TimestampFormatter formatter, String keyName, String longDuplicateKeyName)
         {
-            super(jruby, keyName);
+            super(formatter, keyName);
             timeFieldWriter = new TimestampLongFieldWriter(longDuplicateKeyName);
         }
 

data/src/main/java/org/embulk/output/{TdOutputPlugin.java → td/TdOutputPlugin.java}

@@ -1,7 +1,8 @@
-package org.embulk.output;
+package org.embulk.output.td;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import javax.validation.constraints.Min;
 import javax.validation.constraints.Max;
 
@@ -14,36 +15,34 @@ import com.treasuredata.api.TdApiClientConfig;
 import com.treasuredata.api.TdApiClientConfig.HttpProxyConfig;
 import com.treasuredata.api.TdApiConflictException;
 import com.treasuredata.api.TdApiNotFoundException;
-import com.treasuredata.api.TdApiException;
 import com.treasuredata.api.model.TDBulkImportSession;
 import com.treasuredata.api.model.TDBulkImportSession.ImportStatus;
-import com.treasuredata.api.model.TDDatabase;
 import com.treasuredata.api.model.TDTable;
 import org.embulk.config.CommitReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigException;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
-import org.embulk.output.RecordWriter.FieldWriterSet;
+import org.embulk.output.td.RecordWriter.FieldWriterSet;
 import org.embulk.spi.Exec;
 import org.embulk.spi.ExecSession;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
 import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
-import org.jruby.embed.ScriptingContainer;
 import org.slf4j.Logger;
 
 public class TdOutputPlugin
         implements OutputPlugin
 {
     public interface PluginTask
-            extends Task
+            extends Task, TimestampFormatter.Task
     {
         @Config("apikey")
         public String getApiKey();
@@ -62,6 +61,8 @@ public class TdOutputPlugin
 
         //  TODO connect_timeout, read_timeout, send_timeout
 
+        //  TODO mode[append, replace]
+
         @Config("auto_create_table")
         @ConfigDefault("true")
         public boolean getAutoCreateTable();
@@ -98,8 +99,25 @@ public class TdOutputPlugin
         @ConfigDefault("16384") // default 16MB (unit: kb)
         public long getFileSplitSize();
 
-        @ConfigInject
-        public ScriptingContainer getJRuby();
+        @Override
+        @Config("default_timestamp_format")
+        // SQL timestamp with milliseconds is, by defualt, used because Hive and Presto use
+        // those format. As timestamp type, Presto
+        //   * cannot parse SQL timestamp with timezone like '2015-02-03 04:05:06.789 UTC'
+        //   * cannot parse SQL timestamp with nanoseconds like '2015-02-03 04:05:06.789012345'
+        //   * cannot parse SQL timestamp with microseconds like '2015-02-03 04:05:06.789012'
+        //   * can parse SQL timestamp with milliseconds like '2015-02-03 04:05:06.789'
+        // On the other hand, Hive
+        //   * cannot parse SQL timestamp with timezone like '2015-02-03 04:05:06.789 UTC'
+        //   * can parse SQL timestamp with nanoseconds like '2015-02-03 04:05:06.789012345'
+        //   * can parse SQL timestamp with microseconds like '2015-02-03 04:05:06.789012'
+        //   * can parse SQL timestamp with milliseconds like '2015-02-03 04:05:06.789'
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%3N\"")
+        public String getDefaultTimestampFormat();
+
+        @Config("column_options")
+        @ConfigDefault("{}")
+        public Map<String, TimestampColumnOption> getColumnOptions();
 
         public boolean getDoUpload();
         public void setDoUpload(boolean doUpload);
@@ -108,6 +126,10 @@ public class TdOutputPlugin
         public void setSessionName(String session);
     }
 
+    public interface TimestampColumnOption
+            extends Task, TimestampFormatter.TimestampColumnOption
+    {}
+
     public interface HttpProxyTask
             extends Task
     {
@@ -175,6 +197,13 @@ public class TdOutputPlugin
     {
         final PluginTask task = config.loadConfig(PluginTask.class);
 
+        // TODO mode check
+
+        // check column_options is valid or not
+        for (String columnName : task.getColumnOptions().keySet()) {
+            schema.lookupColumn(columnName); // throws SchemaConfigException
+        }
+
         // generate session name
         task.setSessionName(buildBulkImportSessionName(task, Exec.session()));
 

data/src/test/java/com/treasuredata/api/TestTdApiClient.java

@@ -0,0 +1,79 @@
+package com.treasuredata.api;
+
+import com.treasuredata.api.model.TDDatabase;
+import org.bigtesting.fixd.ServerFixture;
+import org.bigtesting.fixd.core.Method;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class TestTdApiClient
+{
+    private ServerFixture server;
+    private TdApiClient client;
+    private TdApiClientConfig clientConfig;
+    private String apikey = "apikey";
+
+    @Before
+    public void startServer()
+            throws Exception
+    {
+        server = new ServerFixture(9490);
+        server.start();
+    }
+
+    @After
+    public void stopServer()
+            throws Exception
+    {
+        server.stop();
+    }
+
+    @Before
+    public void startTdApiClient()
+            throws Exception
+    {
+        clientConfig = new TdApiClientConfig("localhost:9490", false);
+        client = new TdApiClient(apikey, clientConfig);
+        client.start();
+    }
+
+    @After
+    public void stopTdApiClient()
+            throws Exception
+    {
+        client.close();;
+    }
+
+    private static final String DATABASE_LIST_JSON =
+    "{" +
+        "\"databases\":[" +
+            "{\"name\":\"test1\"}," +
+            "{\"name\":\"test2\"}" +
+        "]" +
+    "}";
+
+    @Test
+    public void getDatabases() throws Exception
+    {
+        server.handle(Method.GET, "/v3/database/list").with(200, "text/json", DATABASE_LIST_JSON);
+        List<TDDatabase> dbs = client.getDatabases();
+        assertEquals(2, dbs.size());
+        assertEquals("test1", dbs.get(0).getName());
+        assertEquals("test2", dbs.get(1).getName());
+    }
+
+    @Test(expected = TdApiNotFoundException.class)
+    public void notFoundDatabases()
+            throws Exception
+    {
+        server.handle(Method.GET, "/v3/database/list").with(404, "text/json", "{\"message\":\"not found\"}");
+        client.getDatabases();
+        fail();
+    }
+}

data/src/test/java/org/embulk/output/{TestTdOutputPlugin.java → td/TestTdOutputPlugin.java}

@@ -1,4 +1,4 @@
-package org.embulk.output;
+package org.embulk.output.td;
 
 public class TestTdOutputPlugin
 {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-td
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-14 00:00:00.000000000 Z
+date: 2015-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,12 +80,13 @@ files:
 - src/main/java/com/treasuredata/api/model/TDTablePermission.java
 - src/main/java/com/treasuredata/api/model/TDTableSchema.java
 - src/main/java/com/treasuredata/api/model/TDTableType.java
-- src/main/java/org/embulk/output/FinalizableExecutorService.java
-- src/main/java/org/embulk/output/MsgpackGZFileBuilder.java
-- src/main/java/org/embulk/output/RecordWriter.java
-- src/main/java/org/embulk/output/TdOutputPlugin.java
-- src/test/java/org/embulk/output/TestTdOutputPlugin.java
-- classpath/embulk-output-td-0.1.2.jar
+- src/main/java/org/embulk/output/td/FinalizableExecutorService.java
+- src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java
+- src/main/java/org/embulk/output/td/RecordWriter.java
+- src/main/java/org/embulk/output/td/TdOutputPlugin.java
+- src/test/java/com/treasuredata/api/TestTdApiClient.java
+- src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
+- classpath/embulk-output-td-0.1.3.jar
 - classpath/javassist-3.18.1-GA.jar
 - classpath/jetty-client-9.2.2.v20140723.jar
 - classpath/jetty-http-9.2.2.v20140723.jar