embulk-filter-add_time 0.1.0

data/src/test/java/org/embulk/filter/add_time/converter/TestSchemaConverter.java

@@ -0,0 +1,338 @@
+package org.embulk.filter.add_time.converter;
+
+import com.google.common.collect.ImmutableMap;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Types;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+
+import static org.embulk.filter.add_time.TestAddTimeFilterPlugin.pluginTask;
+import static org.embulk.filter.add_time.TestAddTimeFilterPlugin.schema;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestSchemaConverter
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private Logger log;
+    private ConfigSource config;
+
+    @Before
+    public void createResources()
+    {
+        log = runtime.getExec().getLogger(TestSchemaConverter.class);
+        config = runtime.getExec().newConfigSource();
+    }
+
+    @Test
+    public void validate()
+    {
+        // if to_column doesn't exist, throws ConfigException
+        {
+            ConfigSource conf = this.config.deepCopy()
+                    .set("from_value", ImmutableMap.of("mode", "upload_time"));
+            failSchemaConverterCreation(log, conf, schema("c0", Types.TIMESTAMP));
+        }
+
+        // if both from_column and from_value exist, throws ConfigException
+        {
+            ConfigSource conf = this.config.deepCopy()
+                    .set("to_column", ImmutableMap.of("name", "time"))
+                    .set("from_column", ImmutableMap.of("name", "c0"))
+                    .set("from_value", ImmutableMap.of("mode", "upload_time"));
+            failSchemaConverterCreation(log, conf, schema("c0", Types.TIMESTAMP));
+        }
+
+        // if from_column or from_value doesn't exist, throws ConfigException
+        {
+            ConfigSource conf = this.config.deepCopy()
+                    .set("to_column", ImmutableMap.of("name", "time"));
+            failSchemaConverterCreation(log, conf, schema("c0", Types.TIMESTAMP));
+        }
+
+        // if from_value type is not string or long, throws ConfigException
+        {
+            ConfigSource conf = this.config.deepCopy()
+                    .set("to_column", ImmutableMap.of("name", "time"))
+                    .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", new ArrayList<Long>()));
+            failSchemaConverterCreation(log, conf, schema("c0", Types.TIMESTAMP));
+        }
+        {
+            ConfigSource conf = this.config.deepCopy()
+                    .set("to_column", ImmutableMap.of("name", "time"))
+                    .set("from_value", ImmutableMap.of("mode", "incremental_time", "from", new ArrayList<Long>(), "to", new ArrayList<Long>()));
+            failSchemaConverterCreation(log, conf, schema("c0", Types.TIMESTAMP));
+        }
+
+        // the type of the column specified as to_column must be 'long' or 'timestamp'
+        { // boolean
+            ConfigSource conf = this.config.deepCopy()
+                    .set("to_column", ImmutableMap.of("name", "time", "type", "boolean"))
+                    .set("from_column", ImmutableMap.of("name", "c0"));
+            failSchemaConverterCreation(log, conf, schema("c0", Types.TIMESTAMP));
+        }
+        { // double
+            ConfigSource conf = this.config.deepCopy()
+                    .set("to_column", ImmutableMap.of("name", "time", "type", "double"))
+                    .set("from_column", ImmutableMap.of("name", "c0"));
+            failSchemaConverterCreation(log, conf, schema("c0", Types.TIMESTAMP));
+        }
+        { // string
+            ConfigSource conf = this.config.deepCopy()
+                    .set("to_column", ImmutableMap.of("name", "time", "type", "string"))
+                    .set("from_column", ImmutableMap.of("name", "c0"));
+            failSchemaConverterCreation(log, conf, schema("c0", Types.TIMESTAMP));
+        }
+
+        // the type of the column specified as from_column must be 'long', 'timestamp' or 'string'
+        {
+            ConfigSource conf = this.config.deepCopy()
+                    .set("to_column", ImmutableMap.of("name", "time"))
+                    .set("from_column", ImmutableMap.of("name", "c0"));
+            failSchemaConverterCreation(log, conf, schema("c0", Types.BOOLEAN)); // boolean
+            failSchemaConverterCreation(log, conf, schema("c0", Types.DOUBLE)); // double
+        }
+    }
+
+    private static void failSchemaConverterCreation(Logger log, ConfigSource conf, Schema inputSchema)
+    {
+        try {
+            new SchemaConverter(log, pluginTask(conf), inputSchema);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof ConfigException);
+        }
+    }
+
+    @Test
+    public void testToColumn()
+    {
+        ConfigSource config = this.config.deepCopy().set("from_value", ImmutableMap.of("mode", "upload_time"));
+
+        // timestamp type
+        { // default type is timestamp
+            ConfigSource conf = config.deepCopy().set("to_column", ImmutableMap.of("name", "time"));
+            Schema inputSchema = schema("c0", Types.TIMESTAMP);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+        {
+            ConfigSource conf = config.deepCopy().set("to_column", ImmutableMap.of("name", "time", "type", "timestamp"));
+            Schema inputSchema = schema("c0", Types.TIMESTAMP);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+
+        // long type
+        { // default unix_timestamp_unit is sec
+            ConfigSource conf = config.deepCopy().set("to_column", ImmutableMap.of("name", "time", "type", "long"));
+            Schema inputSchema = schema("c0", Types.TIMESTAMP);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.LONG), outputSchema.getColumn(1));
+        }
+        { // default unix_timestamp_unit is sec
+            ConfigSource conf = config.deepCopy().set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "milli"));
+            Schema inputSchema = schema("c0", Types.TIMESTAMP);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.LONG), outputSchema.getColumn(1));
+        }
+
+        // renaming
+        { // timestamp type
+            ConfigSource conf = config.deepCopy().set("to_column", ImmutableMap.of("name", "c0"));
+            Schema inputSchema = schema("c0", Types.TIMESTAMP);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(new Column(0, "c0_", Types.TIMESTAMP), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "c0", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+        { // timestamp type
+            ConfigSource conf = config.deepCopy().set("to_column", ImmutableMap.of("name", "c0"));
+            Schema inputSchema = schema("c0", Types.LONG);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(new Column(0, "c0_", Types.LONG), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "c0", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+    }
+
+    @Test
+    public void testFromColumn()
+    {
+        ConfigSource config = this.config.deepCopy().set("to_column", ImmutableMap.of("name", "time"));
+
+        // timestamp type
+        {
+            ConfigSource conf = config.deepCopy().set("from_column", ImmutableMap.of("name", "c0"));
+            Schema inputSchema = schema("c0", Types.TIMESTAMP);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+
+        // long type
+        {
+            ConfigSource conf = config.deepCopy().set("from_column", ImmutableMap.of("name", "c0"));
+            Schema inputSchema = schema("c0", Types.LONG);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+        {
+            ConfigSource conf = config.deepCopy().set("from_column", ImmutableMap.of("name", "c0", "unix_timestamp_unit", "sec"));
+            Schema inputSchema = schema("c0", Types.LONG);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+
+        // string type
+        {
+            ConfigSource conf = config.deepCopy().set("from_column", ImmutableMap.of("name", "c0"));
+            Schema inputSchema = schema("c0", Types.STRING);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+        {
+            ConfigSource conf = config.deepCopy().set("from_column", ImmutableMap.of("name", "c0", "timestamp_format", "%Y-%m-%d %H:%M:%S.%N %Z"));
+            Schema inputSchema = schema("c0", Types.STRING);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+    }
+
+    @Test
+    public void testFromValue()
+    {
+        ConfigSource config = this.config.deepCopy().set("to_column", ImmutableMap.of("name", "time"));
+
+        // mode: fixed_time
+        { // default mode
+            ConfigSource conf = config.deepCopy().set("from_value", ImmutableMap.of("value", "2016-01-01 10:10:10 UTC"));
+            Schema inputSchema = schema("c0", Types.STRING);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+        {
+            ConfigSource conf = config.deepCopy().set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 10:10:10 UTC"));
+            Schema inputSchema = schema("c0", Types.STRING);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+        { // specify 'timestamp_format'
+            ConfigSource conf = config.deepCopy()
+                    .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 10:10:10.000 UTC",
+                            "timestamp_format", "%Y-%m-%d %H:%M:%S.%N %Z"));
+            Schema inputSchema = schema("c0", Types.STRING);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+        { // specify 'unix_timestamp_unit'
+            ConfigSource conf = config.deepCopy()
+                    .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", 1451643010,
+                            "unix_timestamp_unit", "sec"));
+            Schema inputSchema = schema("c0", Types.STRING);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+
+        // mode: incremental_time
+        {
+            ConfigSource conf = config.deepCopy()
+                    .set("from_value", ImmutableMap.of("mode", "incremental_time",
+                            "from", "2016-01-01 10:10:10 UTC", "to", "2016-01-01 10:10:10 UTC"));
+            Schema inputSchema = schema("c0", Types.STRING);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+        { // specify 'timestamp_format'
+            ConfigSource conf = config.deepCopy()
+                    .set("from_value", ImmutableMap.of("mode", "incremental_time",
+                            "from", "2016-01-01 10:10:10.000 UTC", "to", "2016-01-01 10:10:10.000 UTC", "timestamp_format", "%Y-%m-%d %H:%M:%S.%N %Z"));
+            Schema inputSchema = schema("c0", Types.STRING);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+        { // specify 'unix_timestamp_unit'
+            ConfigSource conf = config.deepCopy()
+                    .set("from_value", ImmutableMap.of("mode", "incremental_time",
+                            "from", 1451643010, "to", 1451643010, "unix_timestamp_unit", "sec"));
+            Schema inputSchema = schema("c0", Types.STRING);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+
+        // mode: upload_time
+        {
+            ConfigSource conf = config.deepCopy()
+                    .set("from_value", ImmutableMap.of(
+                            "mode", "upload_time"));
+            Schema inputSchema = schema("c0", Types.STRING);
+            Schema outputSchema = new SchemaConverter(log, pluginTask(conf), inputSchema).toOutputSchema();
+
+            assertEquals(inputSchema.size() + 1, outputSchema.size());
+            assertEquals(inputSchema.getColumn(0), outputSchema.getColumn(0));
+            assertEquals(new Column(1, "time", Types.TIMESTAMP), outputSchema.getColumn(1));
+        }
+    }
+
+}

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: embulk-filter-add_time
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Muga Nishizawa
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-01-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Add time column to the schema
+email:
+- muga.nishizawa@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- CHANGELOG.md
+- COPYING
+- README.md
+- build.gradle
+- embulk-filter-add_time.gemspec
+- gradle/check.gradle
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/filter/add_time.rb
+- src/main/java/org/embulk/filter/add_time/AddTimeFilterPlugin.java
+- src/main/java/org/embulk/filter/add_time/converter/ColumnConverter.java
+- src/main/java/org/embulk/filter/add_time/converter/ColumnDuplicator.java
+- src/main/java/org/embulk/filter/add_time/converter/LongValueCastConverter.java
+- src/main/java/org/embulk/filter/add_time/converter/SchemaConverter.java
+- src/main/java/org/embulk/filter/add_time/converter/SimpleColumnConverter.java
+- src/main/java/org/embulk/filter/add_time/converter/StringValueCastConverter.java
+- src/main/java/org/embulk/filter/add_time/converter/TimestampValueCastConverter.java
+- src/main/java/org/embulk/filter/add_time/converter/ValueCastConverter.java
+- src/main/java/org/embulk/filter/add_time/converter/ValueConverter.java
+- src/main/java/org/embulk/filter/add_time/converter/ValueNoConverter.java
+- src/main/java/org/embulk/filter/add_time/reader/AbstractColumnReader.java
+- src/main/java/org/embulk/filter/add_time/reader/BooleanColumnReader.java
+- src/main/java/org/embulk/filter/add_time/reader/ColumnReader.java
+- src/main/java/org/embulk/filter/add_time/reader/DoubleColumnReader.java
+- src/main/java/org/embulk/filter/add_time/reader/LongColumnReader.java
+- src/main/java/org/embulk/filter/add_time/reader/StringColumnReader.java
+- src/main/java/org/embulk/filter/add_time/reader/TimeValueGenerator.java
+- src/main/java/org/embulk/filter/add_time/reader/TimestampColumnReader.java
+- src/test/java/org/embulk/filter/add_time/TestAddTimeFilterPlugin.java
+- src/test/java/org/embulk/filter/add_time/converter/TestSchemaConverter.java
+- classpath/embulk-filter-add_time-0.1.0.jar
+homepage: https://github.com/treasure-data/embulk-filter-add_time
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Add time filter plugin for Embulk
+test_files: []