embulk-output-td 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f75918c0833bfac5a887d29736ec84c572e9594
-  data.tar.gz: b4079b5ed3740e5f36c3f3c60763549dd2921ff8
+  metadata.gz: 7926b5de5fab7cc6f9343b39e0c5d35f8e2e160e
+  data.tar.gz: 1f581633d6365c1043fef772f856c79e1b29397e
 SHA512:
-  metadata.gz: 80584f78e0fc8c85255e4a10bc53e2e39819253634f22d109e0c5542da4c4e83f7eb43fefc94f9d21ac8cdfe5be6ae3704684f6622b5728f5d6a66c4a9975602
-  data.tar.gz: 5134cd1a02a2c96c8330206d5ced8780ebae83954b825d8bf606b76cf44b9a31bb1801a9133ac5cc4b56c7a5305c66093f1c01839ee36d1113111edf8b2b4d0a
+  metadata.gz: 668eec6a9224c66c7bc0d750af02a8d8a577a584482fa44599722a0478d517ef09b80b214a6821eefee32ebd65ee0dc38d164fb0cd7b3dc13df208d85872aff8
+  data.tar.gz: 94a707125cc4631a6cf71fe4af3a720370cf667374d7eb6e8772a7d478b38b1affea2d44244a91dcef9fb0c47ffb7d0ee67751ce392b622de40dd66e2a1a3d63

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.8 - 2016-01-09
+
+* [new feature] Add mode to time value option  [#31](https://github.com/treasure-data/embulk-output-td/pull/31)
+* [maintenance] Remove unnecessary warning messages within showErrorRecord method [#30](https://github.com/treasure-data/embulk-output-td/pull/30)
+
 ## 0.1.7 - 2016-01-07
 
 * [new feature] Add time_value option  [#16](https://github.com/treasure-data/embulk-output-td/pull/16)

data/build.gradle

@@ -16,7 +16,7 @@ configurations {
     provided
 }
 
-version = "0.1.7"
+version = "0.1.8"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7

data/embulk-output-td.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-td"
-  spec.version       = "0.1.7"
+  spec.version       = "0.1.8"
   spec.authors       = ["Muga Nishizawa"]
   spec.summary       = %[TreasureData output plugin for Embulk]
   spec.description   = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]

data/src/main/java/org/embulk/output/td/TdOutputPlugin.java

@@ -213,19 +213,6 @@ public class TdOutputPlugin
         public boolean getUseSsl();
     }
 
-    public interface TimeValueConfig
-            extends Task
-    {
-        @Config("from")
-        @Min(0)
-        public long getFrom();
-
-        @Config("to")
-        @ConfigDefault("0")
-        @Min(0)
-        public long getTo();
-    }
-
     public static enum ConvertTimestampType
     {
         STRING(-1),
@@ -606,7 +593,9 @@ public class TdOutputPlugin
                 log.info("      - {}: {}", pair.getKey(), pair.getValue());
             }
 
-            showBulkImportErrorRecords(client, sessionName, (int) Math.min(session.getErrorRecords(), task.getDisplayedErrorRecordsCountLimit()));
+            if (session.getErrorRecords() > 0L) {
+                showBulkImportErrorRecords(client, sessionName, (int) Math.min(session.getErrorRecords(), task.getDisplayedErrorRecordsCountLimit()));
+            }
 
             if (session.getErrorRecords() > 0 && task.getStopOnInvalidRecord()) {
                 throw new DataException(String.format("Stop committing because the perform job skipped %d error records", session.getErrorRecords()));

data/src/main/java/org/embulk/output/td/TimeValueConfig.java

@@ -0,0 +1,35 @@
+package org.embulk.output.td;
+
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+
+import javax.validation.constraints.Max;
+import javax.validation.constraints.Min;
+
+public interface TimeValueConfig
+        extends Task
+{
+    @Config("mode")
+    @ConfigDefault("\"incremental_time\"")
+    String getMode();
+
+    @Config("value")
+    @ConfigDefault("null")
+    @Min(0)
+    @Max(253402300799L) // '9999-12-31 23:59:59 UTC'
+    Optional<Long> getValue();
+
+    @Config("from")
+    @ConfigDefault("null")
+    @Min(0)
+    @Max(253402300799L) // '9999-12-31 23:59:59 UTC'
+    Optional<Long> getFrom();
+
+    @Config("to")
+    @ConfigDefault("null")
+    @Min(0)
+    @Max(253402300799L) // '9999-12-31 23:59:59 UTC'
+    Optional<Long> getTo();
+}

data/src/main/java/org/embulk/output/td/TimeValueGenerator.java

@@ -0,0 +1,79 @@
+package org.embulk.output.td;
+
+import com.google.common.base.Optional;
+import org.embulk.config.ConfigException;
+
+public abstract class TimeValueGenerator
+{
+    public abstract long next();
+
+    public static TimeValueGenerator newGenerator(final TimeValueConfig config)
+    {
+        switch (config.getMode()) {
+            case "incremental_time": { // default mode
+                require(config.getFrom(), "'from', 'to'");
+                require(config.getTo(), "'to'");
+                reject(config.getValue(), "'value'");
+
+                return new TimeValueGenerator()
+                {
+                    private final long from = config.getFrom().get();
+                    private final long to = config.getTo().get();
+
+                    private long current = from;
+
+                    @Override
+                    public long next()
+                    {
+                        try {
+                            return current++;
+                        }
+                        finally {
+                            if (current > to) {
+                                current = from;
+                            }
+                        }
+                    }
+                };
+            }
+            case "fixed_time": {
+                require(config.getValue(), "'value'");
+                reject(config.getFrom(), "'from'");
+                reject(config.getTo(), "'to'");
+
+                return new TimeValueGenerator()
+                {
+                    private final long fixed = config.getValue().get();
+
+                    @Override
+                    public long next()
+                    {
+                        return fixed;
+                    }
+                };
+            }
+            default: {
+                throw new ConfigException(String.format("Unknwon mode '%s'. Supported methods are incremental_time, fixed_time.", config.getMode()));
+            }
+        }
+    }
+
+    // ported from embulk-input-s3
+    private static <T> T require(Optional<T> value, String message)
+    {
+        if (value.isPresent()) {
+            return value.get();
+        }
+        else {
+            throw new ConfigException("Required option is not set: " + message);
+        }
+    }
+
+    // ported from embulk-input-s3
+    private static <T> void reject(Optional<T> value, String message)
+    {
+        if (value.isPresent()) {
+            throw new ConfigException("Invalid option is set: " + message);
+        }
+    }
+}

data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java

@@ -7,6 +7,8 @@ import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import org.embulk.config.ConfigException;
 import org.embulk.output.td.TdOutputPlugin;
+import org.embulk.output.td.TimeValueConfig;
+import org.embulk.output.td.TimeValueGenerator;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageReader;
@@ -20,7 +22,6 @@ import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.util.Timestamps;
 import org.embulk.output.td.MsgpackGZFileBuilder;
-import org.embulk.output.td.TdOutputPlugin.TimeValueConfig;
 import org.slf4j.Logger;
 
 public class FieldWriterSet
@@ -220,7 +221,7 @@ public class FieldWriterSet
         }
 
         if (timeValueConfig.isPresent()) {
-            staticTimeValue = Optional.of(new TimeValueGenerator(timeValueConfig.get()));
+            staticTimeValue = Optional.of(TimeValueGenerator.newGenerator(timeValueConfig.get()));
         }
         else {
             staticTimeValue = Optional.absent();
@@ -321,29 +322,4 @@ public class FieldWriterSet
             throw Throwables.propagate(e);
         }
     }
-
-    static class TimeValueGenerator
-    {
-        private final long from;
-        private final long to;
-        private long current;
-
-        TimeValueGenerator(TimeValueConfig config)
-        {
-            current = from = config.getFrom();
-            to = config.getTo();
-        }
-
-        long next()
-        {
-            try {
-                return current++;
-            }
-            finally {
-                if (current >= to) {
-                    current = from;
-                }
-            }
-        }
-    }
 }

data/src/test/java/org/embulk/output/td/TestTimeValueGenerator.java

@@ -0,0 +1,125 @@
+package org.embulk.output.td;
+
+import com.google.common.collect.ImmutableMap;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.output.td.writer.FieldWriterSet;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Types;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+
+import static org.embulk.output.td.TestTdOutputPlugin.config;
+import static org.embulk.output.td.TestTdOutputPlugin.pluginTask;
+import static org.embulk.output.td.TestTdOutputPlugin.schema;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestTimeValueGenerator
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private Logger log;
+    private ConfigSource config;
+    private Schema schema;
+
+    @Before
+    public void createResources()
+    {
+        log = Exec.getLogger(TestTimeValueGenerator.class);
+        config = config();
+    }
+
+    @Test
+    public void validateTimeValue()
+    {
+        // incremental_time
+        { // {from: 0, to: 0} # default incremental_time
+            schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
+            new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "to", 0L))), schema);
+        }
+        { // {from: 0} # default incremental_time
+            schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
+            try {
+                new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L))), schema);
+                fail();
+            }
+            catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+        { // {to: 0} # default incremental_time
+            schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
+            try {
+                new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("to", 0L))), schema);
+                fail();
+            }
+            catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+        { // {from: 0, to: 0, mode: incremental_time}
+            schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
+            new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "to", 0L, "mode", "incremental_time"))), schema);
+        }
+        { // {from: 0, mode: incremental_time}
+            schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
+            try {
+                new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "mode", "incremental_time"))), schema);
+                fail();
+            }
+            catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+        { // {to: 0, mode: incremental_time}
+            schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
+            try {
+                new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("to", 0L, "mode", "incremental_time"))), schema);
+                fail();
+            }
+            catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+        { // {mode: incremental_time}
+            schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
+            try {
+                new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("mode", "incremental_time"))), schema);
+                fail();
+            }
+            catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+
+        // fixed_time
+        { // {value: 0, mode: fixed_time}
+            schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
+            new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("value", 0L, "mode", "fixed_time"))), schema);
+        }
+        { // {mode: fixed_time}
+            schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
+            try {
+                new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("mode", "fixed_time"))), schema);
+            }
+            catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+        { // {value: 0}
+            schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
+            try {
+                new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("value", 0L))), schema);
+            }
+            catch (Throwable t) {
+                assertTrue(t instanceof ConfigException);
+            }
+        }
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-td
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-07 00:00:00.000000000 Z
+date: 2016-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -87,6 +87,8 @@ files:
 - src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java
 - src/main/java/org/embulk/output/td/RecordWriter.java
 - src/main/java/org/embulk/output/td/TdOutputPlugin.java
+- src/main/java/org/embulk/output/td/TimeValueConfig.java
+- src/main/java/org/embulk/output/td/TimeValueGenerator.java
 - src/main/java/org/embulk/output/td/writer/BooleanFieldWriter.java
 - src/main/java/org/embulk/output/td/writer/DoubleFieldWriter.java
 - src/main/java/org/embulk/output/td/writer/FieldWriter.java
@@ -102,8 +104,9 @@ files:
 - src/test/java/com/treasuredata/api/TestTdApiClient.java
 - src/test/java/org/embulk/output/td/TestRecordWriter.java
 - src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
+- src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
 - src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
-- classpath/embulk-output-td-0.1.7.jar
+- classpath/embulk-output-td-0.1.8.jar
 - classpath/javassist-3.18.1-GA.jar
 - classpath/jetty-client-9.2.2.v20140723.jar
 - classpath/jetty-http-9.2.2.v20140723.jar