embulk-output-td 0.1.8 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7926b5de5fab7cc6f9343b39e0c5d35f8e2e160e
-  data.tar.gz: 1f581633d6365c1043fef772f856c79e1b29397e
+  metadata.gz: 985479cb77ed1b156d896d1dbad073fdc6141c56
+  data.tar.gz: 4fde290d7d7e83a7eba0f687c22312e0ad589f8d
 SHA512:
-  metadata.gz: 668eec6a9224c66c7bc0d750af02a8d8a577a584482fa44599722a0478d517ef09b80b214a6821eefee32ebd65ee0dc38d164fb0cd7b3dc13df208d85872aff8
-  data.tar.gz: 94a707125cc4631a6cf71fe4af3a720370cf667374d7eb6e8772a7d478b38b1affea2d44244a91dcef9fb0c47ffb7d0ee67751ce392b622de40dd66e2a1a3d63
+  metadata.gz: 87aba00cd303cbe98772d3e645d2336fc15091f502829eaed3064def64ac4154bc4d39110d785f30fd046e34a183a4bc131953c5d1524732f72893ddea8c9f66
+  data.tar.gz: a3a2281e0bcd43ef99763ae8ab7e526f8d22b1f667bb480e5c16b52bf5bcbf6cca4caeedcf5fbf35b1bd801a1cc5c8896b55f65007d522c4f877373818bbfebd

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.0 - 2016-01-12
+
+* [new feature] Not use first timestamp column as primary key [#32](https://github.com/treasure-data/embulk-output-td/pull/32)
+
 ## 0.1.8 - 2016-01-09
 
 * [new feature] Add mode to time value option  [#31](https://github.com/treasure-data/embulk-output-td/pull/31)

data/build.gradle

@@ -16,7 +16,7 @@ configurations {
     provided
 }
 
-version = "0.1.8"
+version = "0.2.0"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7

data/embulk-output-td.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-td"
-  spec.version       = "0.1.8"
+  spec.version       = "0.2.0"
   spec.authors       = ["Muga Nishizawa"]
   spec.summary       = %[TreasureData output plugin for Embulk]
   spec.description   = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]

data/src/main/java/org/embulk/output/td/TdOutputPlugin.java

@@ -109,6 +109,7 @@ public class TdOutputPlugin
         @Config("time_value")
         @ConfigDefault("null")
         public Optional<TimeValueConfig> getTimeValue(); // TODO allow timestamp format such as {from: "2015-01-01 00:00:00 UTC", to: "2015-01-02 00:00:00 UTC"} as well as unixtime integer
+        public void setTimeValue(Optional<TimeValueConfig> timeValue);
 
         @Config("unix_timestamp_unit")
         @ConfigDefault("\"sec\"")

data/src/main/java/org/embulk/output/td/TimeValueConfig.java

@@ -5,9 +5,6 @@ import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.Task;
 
-import javax.validation.constraints.Max;
-import javax.validation.constraints.Min;
-
 public interface TimeValueConfig
         extends Task
 {
@@ -17,19 +14,13 @@ public interface TimeValueConfig
 
     @Config("value")
     @ConfigDefault("null")
-    @Min(0)
-    @Max(253402300799L) // '9999-12-31 23:59:59 UTC'
     Optional<Long> getValue();
 
     @Config("from")
     @ConfigDefault("null")
-    @Min(0)
-    @Max(253402300799L) // '9999-12-31 23:59:59 UTC'
     Optional<Long> getFrom();
 
     @Config("to")
     @ConfigDefault("null")
-    @Min(0)
-    @Max(253402300799L) // '9999-12-31 23:59:59 UTC'
     Optional<Long> getTo();
 }

data/src/main/java/org/embulk/output/td/TimeValueGenerator.java

@@ -10,54 +10,73 @@ public abstract class TimeValueGenerator
     public static TimeValueGenerator newGenerator(final TimeValueConfig config)
     {
         switch (config.getMode()) {
-            case "incremental_time": { // default mode
+            case "incremental_time": // default mode
                 require(config.getFrom(), "'from', 'to'");
+                validateTimeRange(config.getFrom().get(), "'from'");
                 require(config.getTo(), "'to'");
+                validateTimeRange(config.getTo().get(), "'to'");
                 reject(config.getValue(), "'value'");
 
-                return new TimeValueGenerator()
-                {
-                    private final long from = config.getFrom().get();
-                    private final long to = config.getTo().get();
-
-                    private long current = from;
-
-                    @Override
-                    public long next()
-                    {
-                        try {
-                            return current++;
-                        }
-                        finally {
-                            if (current > to) {
-                                current = from;
-                            }
-                        }
-                    }
-                };
-            }
-            case "fixed_time": {
+                return new IncrementalTimeValueGenerator(config);
+
+            case "fixed_time":
                 require(config.getValue(), "'value'");
+                validateTimeRange(config.getValue().get(), "'value'");
                 reject(config.getFrom(), "'from'");
                 reject(config.getTo(), "'to'");
 
-                return new TimeValueGenerator()
-                {
-                    private final long fixed = config.getValue().get();
+                return new FixedTimeValueGenerator(config);
 
-                    @Override
-                    public long next()
-                    {
-                        return fixed;
-                    }
-                };
-            }
-            default: {
+            default:
                 throw new ConfigException(String.format("Unknwon mode '%s'. Supported methods are incremental_time, fixed_time.", config.getMode()));
+        }
+    }
+
+    public static class IncrementalTimeValueGenerator
+            extends TimeValueGenerator
+    {
+        private final long from;
+        private final long to;
+
+        private long current;
+
+        public IncrementalTimeValueGenerator(final TimeValueConfig config)
+        {
+            current = from = config.getFrom().get();
+            to = config.getTo().get();
+        }
+
+        @Override
+        public long next()
+        {
+            try {
+                return current++;
+            }
+            finally {
+                if (current > to) {
+                    current = from;
+                }
             }
         }
     }
 
+    public static class FixedTimeValueGenerator
+            extends TimeValueGenerator
+    {
+        private final long value;
+
+        public FixedTimeValueGenerator(final TimeValueConfig config)
+        {
+            value = config.getValue().get();
+        }
+
+        @Override
+        public long next()
+        {
+            return value;
+        }
+    }
+
     // ported from embulk-input-s3
     private static <T> T require(Optional<T> value, String message)
     {
@@ -69,6 +88,13 @@ public abstract class TimeValueGenerator
         }
     }
 
+    private static void validateTimeRange(long value, String message)
+    {
+        if (value < 0 || 253402300799L < value) { // should be [1970-01-01 00:00:00, 9999-12-31 23:59:59]
+            throw new ConfigException("The option value must be within [0, 253402300799L]: " + message);
+        }
+    }
+
     // ported from embulk-input-s3
     private static <T> void reject(Optional<T> value, String message)
     {

data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java

@@ -6,11 +6,14 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
 import org.embulk.output.td.TdOutputPlugin;
+import org.embulk.output.td.TdOutputPlugin.ConvertTimestampType;
 import org.embulk.output.td.TimeValueConfig;
 import org.embulk.output.td.TimeValueGenerator;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Exec;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.time.TimestampFormatter;
@@ -40,15 +43,14 @@ public class FieldWriterSet
     public FieldWriterSet(Logger log, TdOutputPlugin.PluginTask task, Schema schema)
     {
         Optional<String> userDefinedPrimaryKeySourceColumnName = task.getTimeColumn();
-        TdOutputPlugin.ConvertTimestampType convertTimestamp = task.getConvertTimestampType();
+        ConvertTimestampType convertTimestampType = task.getConvertTimestampType();
         Optional<TimeValueConfig> timeValueConfig = task.getTimeValue();
         if (timeValueConfig.isPresent() && userDefinedPrimaryKeySourceColumnName.isPresent()) {
             throw new ConfigException("Setting both time_column and time_value is invalid");
         }
 
-        boolean hasPkWriter = false;
+        boolean foundPrimaryKey = false;
         int duplicatePrimaryKeySourceIndex = -1;
-        int firstTimestampColumnIndex = -1;
 
         int fc = 0;
         fieldWriters = new IFieldWriter[schema.size()];
@@ -104,12 +106,11 @@ public class FieldWriterSet
                             log.warn("time column is converted from {} to seconds", task.getUnixTimestampUnit());
                         }
                         writer = new UnixTimestampLongFieldWriter(columnName, task.getUnixTimestampUnit().getFractionUnit());
-                        hasPkWriter = true;
+                        foundPrimaryKey = true;
                     }
                     else if (columnType instanceof TimestampType) {
                         writer = new TimestampLongFieldWriter(columnName);
-
-                        hasPkWriter = true;
+                        foundPrimaryKey = true;
                     }
                     else {
                         throw new ConfigException(String.format("Type of '%s' column must be long or timestamp but got %s",
@@ -118,37 +119,7 @@ public class FieldWriterSet
                     break;
 
                 case SIMPLE_VALUE:
-                    if (columnType instanceof BooleanType) {
-                        writer = new BooleanFieldWriter(columnName);
-                    }
-                    else if (columnType instanceof LongType) {
-                        writer = new LongFieldWriter(columnName);
-                    }
-                    else if (columnType instanceof DoubleType) {
-                        writer = new DoubleFieldWriter(columnName);
-                    }
-                    else if (columnType instanceof StringType) {
-                        writer = new StringFieldWriter(columnName);
-                    }
-                    else if (columnType instanceof TimestampType) {
-                        switch (convertTimestamp) {
-                        case STRING:
-                            writer = new TimestampStringFieldWriter(timestampFormatters[i], columnName);
-                            break;
-                        case SEC:
-                            writer = new TimestampLongFieldWriter(columnName);
-                            break;
-                        default:
-                            // Thread of control doesn't come here but, just in case, it throws ConfigException.
-                            throw new ConfigException(String.format("Unknown option {} as convert_timestamp_type", convertTimestamp));
-                        }
-                        if (firstTimestampColumnIndex < 0) {
-                            firstTimestampColumnIndex = i;
-                        }
-                    }
-                    else {
-                        throw new ConfigException("Unsupported type: " + columnType);
-                    }
+                    writer = newSimpleFieldWriter(columnName, columnType, convertTimestampType, timestampFormatters[i]);
                     break;
 
                 case DUPLICATE_PRIMARY_KEY:
@@ -164,24 +135,24 @@ public class FieldWriterSet
             fc += 1;
         }
 
+        if (foundPrimaryKey) {
+            // appropriate 'time' column is found
+
+            staticTimeValue = Optional.absent();
+            fieldCount = fc;
+            return;
+        }
+
         if (timeValueConfig.isPresent()) {
-            // "time" column is written by RecordWriter
-            fc += 1;
+            // 'time_value' option is specified
+
+            staticTimeValue = Optional.of(TimeValueGenerator.newGenerator(timeValueConfig.get()));
+            fieldCount = fc + 1;
+            return;
         }
-        else if (!hasPkWriter) {
-            // PRIMARY_KEY was not found.
-            if (duplicatePrimaryKeySourceIndex < 0) {
-                if (userDefinedPrimaryKeySourceColumnName.isPresent()) {
-                    throw new ConfigException(String.format("time_column '%s' does not exist", userDefinedPrimaryKeySourceColumnName.get()));
-                }
-                else if (firstTimestampColumnIndex >= 0) {
-                    // if time is not found, use the first timestamp column
-                    duplicatePrimaryKeySourceIndex = firstTimestampColumnIndex;
-                }
-                else {
-                    throw new ConfigException(String.format("TD output plugin requires at least one timestamp column, or a long column named 'time'"));
-                }
-            }
+
+        if (!foundPrimaryKey && duplicatePrimaryKeySourceIndex >= 0) {
+            // 'time_column' option is correctly specified
 
             String columnName = schema.getColumnName(duplicatePrimaryKeySourceIndex);
             Type columnType = schema.getColumnType(duplicatePrimaryKeySourceIndex);
@@ -196,18 +167,7 @@ public class FieldWriterSet
             else if (columnType instanceof TimestampType) {
                 log.info("Duplicating {}:{} column to 'time' column as seconds for the data partitioning",
                         columnName, columnType);
-                IFieldWriter fw;
-                switch (convertTimestamp) {
-                    case STRING:
-                        fw = new TimestampStringFieldWriter(timestampFormatters[duplicatePrimaryKeySourceIndex], columnName);
-                        break;
-                    case SEC:
-                        fw = new TimestampLongFieldWriter(columnName);
-                        break;
-                    default:
-                        // Thread of control doesn't come here but, just in case, it throws ConfigException.
-                        throw new ConfigException(String.format("Unknown option {} as convert_timestamp_type", convertTimestamp));
-                }
+                IFieldWriter fw = newSimpleTimestampFieldWriter(columnName, columnType, convertTimestampType, timestampFormatters[duplicatePrimaryKeySourceIndex]);
                 writer = new TimestampFieldLongDuplicator(fw, "time");
             }
             else {
@@ -217,17 +177,28 @@ public class FieldWriterSet
 
             // replace existint writer
             fieldWriters[duplicatePrimaryKeySourceIndex] = writer;
-            fc += 1;
+            staticTimeValue = Optional.absent();
+            fieldCount = fc + 1;
+            return;
         }
 
-        if (timeValueConfig.isPresent()) {
-            staticTimeValue = Optional.of(TimeValueGenerator.newGenerator(timeValueConfig.get()));
-        }
-        else {
-            staticTimeValue = Optional.absent();
+        if (!foundPrimaryKey) {
+            // primary key is not found yet
+
+            if (userDefinedPrimaryKeySourceColumnName.isPresent()) {
+                throw new ConfigException(String.format("A specified time_column '%s' does not exist", userDefinedPrimaryKeySourceColumnName.get()));
+            }
+
+            long uploadTime = System.currentTimeMillis() / 1000;
+            log.info("'time' column is generated and is set to a unix time {}", uploadTime);
+            TimeValueConfig newConfig = Exec.newConfigSource().set("mode", "fixed_time").set("value", uploadTime).loadConfig(TimeValueConfig.class);
+            task.setTimeValue(Optional.of(newConfig));
+            staticTimeValue = Optional.of(TimeValueGenerator.newGenerator(newConfig));
+            fieldCount = fc + 1;
+            return;
         }
 
-        fieldCount = fc;
+        throw new AssertionError("Cannot select primary key");
     }
 
     private static String newColumnUniqueName(String originalName, Schema schema)
@@ -250,6 +221,43 @@ public class FieldWriterSet
         return false;
     }
 
+    private static FieldWriter newSimpleFieldWriter(String columnName, Type columnType, ConvertTimestampType convertTimestampType, TimestampFormatter timestampFormatter)
+    {
+        if (columnType instanceof BooleanType) {
+            return new BooleanFieldWriter(columnName);
+        }
+        else if (columnType instanceof LongType) {
+            return new LongFieldWriter(columnName);
+        }
+        else if (columnType instanceof DoubleType) {
+            return new DoubleFieldWriter(columnName);
+        }
+        else if (columnType instanceof StringType) {
+            return new StringFieldWriter(columnName);
+        }
+        else if (columnType instanceof TimestampType) {
+            return newSimpleTimestampFieldWriter(columnName, columnType, convertTimestampType, timestampFormatter);
+        }
+        else {
+            throw new ConfigException("Unsupported type: " + columnType);
+        }
+    }
+
+    private static FieldWriter newSimpleTimestampFieldWriter(String columnName, Type columnType, ConvertTimestampType convertTimestampType, TimestampFormatter timestampFormatter)
+    {
+        switch (convertTimestampType) {
+        case STRING:
+            return new TimestampStringFieldWriter(timestampFormatter, columnName);
+
+        case SEC:
+            return new TimestampLongFieldWriter(columnName);
+
+        default:
+            // Thread of control doesn't come here but, just in case, it throws ConfigException.
+            throw new ConfigException(String.format("Unknown option {} as convert_timestamp_type", convertTimestampType));
+        }
+    }
+
     @VisibleForTesting
     public IFieldWriter getFieldWriter(int index)
     {

data/src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java

@@ -48,17 +48,6 @@ public class TestFieldWriterSet
             }
         }
 
-        { // if schema doesn't have time column and the user doesn't specify time_column option, it throws ConfigError.
-            schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
-            try {
-                new FieldWriterSet(log, pluginTask(config), schema);
-                fail();
-            }
-            catch (Throwable t) {
-                assertTrue(t instanceof ConfigException);
-            }
-        }
-
         { // if schema doesn't have a column specified as time_column column, it throws ConfigError
             schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
             try {
@@ -205,6 +194,7 @@ public class TestFieldWriterSet
         Schema schema = schema("_c0", Types.TIMESTAMP, "_c1", Types.LONG);
         FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema);
 
-        assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator);
+        assertTrue(writers.getFieldWriter(0) instanceof TimestampStringFieldWriter); // c0
+        assertTrue(writers.getFieldWriter(1) instanceof LongFieldWriter); // c1
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-td
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.0
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-10 00:00:00.000000000 Z
+date: 2016-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ files:
 - src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
 - src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
 - src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
-- classpath/embulk-output-td-0.1.8.jar
+- classpath/embulk-output-td-0.2.0.jar
 - classpath/javassist-3.18.1-GA.jar
 - classpath/jetty-client-9.2.2.v20140723.jar
 - classpath/jetty-http-9.2.2.v20140723.jar