embulk-filter-add_time 0.1.0

data/gradlew.bat

@@ -0,0 +1,90 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/filter/add_time.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_filter(
+  "add_time", "org.embulk.filter.add_time.AddTimeFilterPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/filter/add_time/AddTimeFilterPlugin.java

@@ -0,0 +1,208 @@
+package org.embulk.filter.add_time;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.filter.add_time.converter.SchemaConverter;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.time.Timestamp;
+import org.slf4j.Logger;
+
+public class AddTimeFilterPlugin
+        implements FilterPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("to_column")
+        ToColumnConfig getToColumn();
+
+        @Config("from_column")
+        @ConfigDefault("null")
+        Optional<FromColumnConfig> getFromColumn();
+
+        @Config("from_value")
+        @ConfigDefault("null")
+        Optional<FromValueConfig> getFromValue();
+    }
+
+    public interface ToColumnConfig
+            extends Task
+    {
+        @Config("name")
+        String getName();
+
+        @Config("type")
+        @ConfigDefault("\"timestamp\"")
+        String getType();
+
+        @Config("unix_timestamp_unit")
+        @ConfigDefault("\"sec\"")
+        String getUnixTimestampUnit();
+    }
+
+    public interface FromColumnConfig
+            extends Task, org.embulk.spi.time.TimestampParser.Task, org.embulk.spi.time.TimestampParser.TimestampColumnOption
+    {
+        @Config("name")
+        String getName();
+
+        @Config("unix_timestamp_unit")
+        @ConfigDefault("\"sec\"")
+        String getUnixTimestampUnit();
+
+        @Config("timestamp_format")
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S %z\"") // override default value
+        Optional<String> getFormat();
+
+        @Config("default_timestamp_format")
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S %z\"") // override default value
+        String getDefaultTimestampFormat();
+    }
+
+    public interface FromValueConfig
+            extends Task, org.embulk.spi.time.TimestampParser.Task, org.embulk.spi.time.TimestampParser.TimestampColumnOption
+    {
+        @Config("mode")
+        @ConfigDefault("\"fixed_time\"")
+        String getMode();
+
+        @Config("value")
+        @ConfigDefault("null")
+        Optional<Object> getValue();
+
+        @Config("from")
+        @ConfigDefault("null")
+        Optional<Object> getFrom();
+
+        @Config("to")
+        @ConfigDefault("null")
+        Optional<Object> getTo();
+
+        @Config("unix_timestamp_unit")
+        @ConfigDefault("\"sec\"")
+        String getUnixTimestampUnit();
+
+        @Config("timestamp_format")
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S %z\"") // override default value
+        Optional<String> getFormat();
+
+        @Config("default_timestamp_format")
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S %z\"") // override default value
+        String getDefaultTimestampFormat();
+    }
+
+    public enum UnixTimestampUnit
+    {
+        SEC(1, 1000000000),
+        MILLI(1000, 1000000),
+        MICRO(1000000, 1000),
+        NANO(1000000000, 1);
+
+        private final int secondUnit;
+        private final int nanoUnit;
+
+        UnixTimestampUnit(int secondUnit, int nanoUnit)
+        {
+            this.secondUnit = secondUnit;
+            this.nanoUnit = nanoUnit;
+        }
+
+        public long toLong(Timestamp t)
+        {
+            return t.getEpochSecond() * secondUnit + t.getNano() / nanoUnit;
+        }
+
+        public Timestamp toTimestamp(long t)
+        {
+            return Timestamp.ofEpochSecond(t / secondUnit, (int) (t % secondUnit * nanoUnit));
+        }
+
+        public static UnixTimestampUnit of(String s)
+        {
+            switch (s) {
+            case "sec": return SEC;
+            case "milli": return MILLI;
+            case "micro": return MICRO;
+            case "nano": return NANO;
+            default:
+                throw new ConfigException(
+                        String.format("Unknown unix_timestamp_unit '%s'. Supported units are sec, milli, micro, and nano", s));
+            }
+        }
+    }
+
+    private final Logger log;
+
+    public AddTimeFilterPlugin()
+    {
+        this.log = Exec.getLogger(getClass());
+    }
+
+    @Override
+    public void transaction(ConfigSource config, Schema inputSchema,
+            FilterPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        control.run(task.dump(), new SchemaConverter(log, task, inputSchema).toOutputSchema());
+    }
+
+    @Override
+    public PageOutput open(TaskSource taskSource, Schema inputSchema,
+            Schema outputSchema, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        return new PageConverter(log, inputSchema, outputSchema, output, new SchemaConverter(log, task, inputSchema));
+    }
+
+    static class PageConverter
+            implements PageOutput
+    {
+        private final Logger log;
+        private SchemaConverter schemaConverter;
+        private final PageReader pageReader;
+        private final PageBuilder pageBuilder;
+
+        public PageConverter(Logger log, Schema inputSchema, Schema outputSchema, PageOutput output, SchemaConverter schemaConverter)
+        {
+            this.log = log;
+            this.schemaConverter = schemaConverter;
+            this.pageReader = new PageReader(inputSchema);
+            this.pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
+        }
+
+        @Override
+        public void add(Page page)
+        {
+            pageReader.setPage(page);
+
+            while (pageReader.nextRecord()) {
+                schemaConverter.convertRecord(pageReader, pageBuilder);
+            }
+        }
+
+        @Override
+        public void finish()
+        {
+            pageBuilder.finish();
+        }
+
+        @Override
+        public void close()
+        {
+            pageBuilder.close();
+        }
+    }
+
+}

data/src/main/java/org/embulk/filter/add_time/converter/ColumnConverter.java

@@ -0,0 +1,14 @@
+package org.embulk.filter.add_time.converter;
+
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+
+public interface ColumnConverter
+{
+    void update(PageReader pageReader);
+
+    void convert(PageBuilder pageBuilder);
+
+    void addColumn(Schema.Builder schemaBuilder);
+}

data/src/main/java/org/embulk/filter/add_time/converter/ColumnDuplicator.java

@@ -0,0 +1,72 @@
+package org.embulk.filter.add_time.converter;
+
+import org.embulk.filter.add_time.reader.ColumnReader;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+
+public class ColumnDuplicator
+        implements ColumnConverter
+{
+    public static class Builder
+    {
+        private Column column;
+        private ColumnReader duplicator;
+        private ColumnReader duplicatee;
+
+        public Builder()
+        {
+        }
+
+        public Builder setColumn(Column column)
+        {
+            this.column = column;
+            return this;
+        }
+
+        public Builder setDuplicator(ColumnReader duplicator)
+        {
+            this.duplicator = duplicator;
+            return this;
+        }
+
+        public Builder setDuplicatee(ColumnReader duplicatee)
+        {
+            this.duplicatee = duplicatee;
+            return this;
+        }
+
+        public ColumnDuplicator build()
+        {
+            return new ColumnDuplicator(column, duplicator, duplicatee);
+        }
+    }
+
+    private final Column column;
+    private final ColumnReader duplicator;
+    private final ColumnReader duplicatee;
+
+    private ColumnDuplicator(Column column, ColumnReader duplicator, ColumnReader duplicatee)
+    {
+        this.column = column;
+        this.duplicator = duplicator;
+        this.duplicatee = duplicatee;
+    }
+
+    public void update(PageReader pageReader)
+    {
+        duplicator.readValue(column, pageReader);
+        duplicator.copyTo(duplicatee);
+    }
+
+    public void convert(PageBuilder pageBuilder)
+    {
+        duplicator.convertValue(column, pageBuilder);
+    }
+
+    public void addColumn(Schema.Builder schemaBuilder)
+    {
+        schemaBuilder.add(column.getName(), column.getType());
+    }
+}

data/src/main/java/org/embulk/filter/add_time/converter/LongValueCastConverter.java

@@ -0,0 +1,33 @@
+package org.embulk.filter.add_time.converter;
+
+import org.embulk.filter.add_time.AddTimeFilterPlugin.FromColumnConfig;
+import org.embulk.filter.add_time.AddTimeFilterPlugin.ToColumnConfig;
+import org.embulk.filter.add_time.AddTimeFilterPlugin.UnixTimestampUnit;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+
+public class LongValueCastConverter
+        extends ValueCastConverter
+{
+    private final UnixTimestampUnit fromUnixTimestampUnit;
+
+    public LongValueCastConverter(FromColumnConfig fromColumnConfig, ToColumnConfig toColumnConfig)
+    {
+        super(toColumnConfig);
+        this.fromUnixTimestampUnit = UnixTimestampUnit.of(fromColumnConfig.getUnixTimestampUnit());
+    }
+
+    @Override
+    public void convertValue(final Column column, long value, final PageBuilder pageBuilder)
+    {
+        columnVisitor.setValue(longToTimestamp(value));
+        columnVisitor.setPageBuilder(pageBuilder);
+        column.visit(columnVisitor);
+    }
+
+    private Timestamp longToTimestamp(long value)
+    {
+        return fromUnixTimestampUnit.toTimestamp(value);
+    }
+}

data/src/main/java/org/embulk/filter/add_time/converter/SchemaConverter.java

@@ -0,0 +1,257 @@
+package org.embulk.filter.add_time.converter;
+
+import com.google.common.base.Optional;
+import org.embulk.config.ConfigException;
+import org.embulk.filter.add_time.AddTimeFilterPlugin.PluginTask;
+import org.embulk.filter.add_time.AddTimeFilterPlugin.FromColumnConfig;
+import org.embulk.filter.add_time.AddTimeFilterPlugin.FromValueConfig;
+import org.embulk.filter.add_time.AddTimeFilterPlugin.ToColumnConfig;
+import org.embulk.filter.add_time.reader.BooleanColumnReader;
+import org.embulk.filter.add_time.reader.ColumnReader;
+import org.embulk.filter.add_time.reader.DoubleColumnReader;
+import org.embulk.filter.add_time.reader.LongColumnReader;
+import org.embulk.filter.add_time.reader.StringColumnReader;
+import org.embulk.filter.add_time.reader.TimeValueGenerator;
+import org.embulk.filter.add_time.reader.TimestampColumnReader;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.DataException;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.BooleanType;
+import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.type.StringType;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.slf4j.Logger;
+
+public class SchemaConverter
+{
+    private final Logger log;
+    private final ColumnConverter[] converters;
+
+    public SchemaConverter(Logger log, PluginTask task, Schema inputSchema)
+    {
+        this.log = log;
+
+        ToColumnConfig toColumnConfig = task.getToColumn();
+        final String toColumnName = toColumnConfig.getName();
+        final Type toColumnType = toToColumnType(toColumnName, toColumnConfig.getType()); // TODO getType should return Type object
+
+        Optional<FromColumnConfig> fromColumnConfig = task.getFromColumn();
+        Optional<FromValueConfig> fromValueConfig = task.getFromValue();
+
+        if (fromColumnConfig.isPresent() && fromValueConfig.isPresent()) {
+            throw new ConfigException("Setting both from_column and from_value is invalid.");
+        }
+        if (!fromColumnConfig.isPresent() && !fromValueConfig.isPresent()) {
+            throw new ConfigException("Setting from_column or from_value is required.");
+        }
+
+        converters = new ColumnConverter[inputSchema.size() + 1];
+
+        for (int i = 0; i < inputSchema.size(); i++) {
+            Column column = inputSchema.getColumn(i);
+            String columnName = column.getName();
+            Type columnType = column.getType();
+
+            final String newColumnName;
+            if (columnName.equals(toColumnName)) {
+                newColumnName = newColumnUniqueName(columnName, inputSchema);
+                log.warn("to_column '{}' is set but '{}' column also exists. The existent '{}' column is renamed to '{}'.",
+                        toColumnName, toColumnName, toColumnName, newColumnName);
+            }
+            else {
+                newColumnName = columnName;
+            }
+
+            if (fromColumnConfig.isPresent() && columnName.equals(fromColumnConfig.get().getName())) {
+                if (!columnType.equals(Types.LONG) && !columnType.equals(Types.STRING) && !columnType.equals(Types.TIMESTAMP)) {
+                    throw new ConfigException(String.format(
+                            "The type of the '%s' column specified as from_column must be long, string or timestamp. But it's %s.", columnName, columnType));
+                }
+
+                ColumnReader duplicatee = newColumnReader(columnType, newValueCastConverter(columnType, fromColumnConfig, toColumnConfig));
+                converters[inputSchema.size()] = new SimpleColumnConverter.Builder()
+                        .setColumn(new Column(inputSchema.size(), toColumnName, toColumnType))
+                        .setColumnReader(duplicatee)
+                        .build();
+                converters[i] = new ColumnDuplicator.Builder()
+                        .setColumn(new Column(i, newColumnName, columnType))
+                        .setDuplicator(newColumnReader(columnType, ValueConverter.NO_CONV))
+                        .setDuplicatee(duplicatee)
+                        .build();
+            }
+            else {
+                converters[i] = new SimpleColumnConverter.Builder()
+                        .setColumn(new Column(i, newColumnName, columnType))
+                        .setColumnReader(newColumnReader(columnType, ValueConverter.NO_CONV))
+                        .build();
+            }
+        }
+
+        if (fromValueConfig.isPresent()) {
+            // create column converter for from_value
+            converters[inputSchema.size()] = new SimpleColumnConverter.Builder()
+                    .setColumn(new Column(inputSchema.size(), toColumnName, toColumnType))
+                    .setColumnReader(TimeValueGenerator.newGenerator(fromValueConfig.get(), newValueCastConverter(Types.TIMESTAMP, fromColumnConfig, toColumnConfig)))
+                    .build();
+        }
+    }
+
+    private static String newColumnUniqueName(String originalName, Schema schema)
+    {
+        String name = originalName;
+        do {
+            name += "_";
+        }
+        while (containsColumnName(schema, name));
+        return name;
+    }
+
+    private static boolean containsColumnName(Schema schema, String name)
+    {
+        for (Column c : schema.getColumns()) {
+            if (c.getName().equals(name)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private static ColumnReader newColumnReader(Type columnType, ValueConverter valueConverter)
+    {
+        if (columnType instanceof BooleanType) {
+            return new BooleanColumnReader(valueConverter);
+        }
+        else if (columnType instanceof LongType) {
+            return new LongColumnReader(valueConverter);
+        }
+        else if (columnType instanceof DoubleType) {
+            return new DoubleColumnReader(valueConverter);
+        }
+        else if (columnType instanceof StringType) {
+            return new StringColumnReader(valueConverter);
+        }
+        else if (columnType instanceof TimestampType) {
+            return new TimestampColumnReader(valueConverter);
+        }
+        // TODO support Json type
+        else {
+            throw new ConfigException("Unsupported type: " + columnType); // TODO after json type support, it should be changed to AssertionError.
+        }
+    }
+
+    private static ValueCastConverter newValueCastConverter(Type columnType, Optional<FromColumnConfig> fromColumnConfig, ToColumnConfig toColumnConfig)
+    {
+        if (columnType instanceof LongType) {
+            return new LongValueCastConverter(fromColumnConfig.get(), toColumnConfig);
+        }
+        else if (columnType instanceof StringType) {
+            return new StringValueCastConverter(fromColumnConfig.get(), toColumnConfig);
+        }
+        else if (columnType instanceof TimestampType) {
+            return new TimestampValueCastConverter(toColumnConfig);
+        }
+        else {
+            throw new AssertionError("Unsupported type: " + columnType);
+        }
+    }
+
+    private static Type toToColumnType(String name, String type)
+    {
+        switch (type) {
+            case "long":
+                return Types.LONG;
+            case "timestamp":
+                return Types.TIMESTAMP;
+            default:
+                throw new ConfigException(String.format( // TODO should return AssertionError
+                        "The type of the '{}' column specified as to_column must be long or timestamp. But it's {}.", name, type));
+        }
+    }
+
+    public void convertRecord(final PageReader pageReader, final PageBuilder pageBuilder)
+    {
+        try {
+            beginRecordConversion();
+
+            pageReader.getSchema().visitColumns(new ColumnVisitor()
+            {
+                @Override
+                public void booleanColumn(Column column)
+                {
+                    updateColumn(column, pageReader);
+                }
+
+                @Override
+                public void longColumn(Column column)
+                {
+                    updateColumn(column, pageReader);
+                }
+
+                @Override
+                public void doubleColumn(Column column)
+                {
+                    updateColumn(column, pageReader);
+                }
+
+                @Override
+                public void stringColumn(Column column)
+                {
+                    updateColumn(column, pageReader);
+                }
+
+                @Override
+                public void timestampColumn(Column column)
+                {
+                    updateColumn(column, pageReader);
+                }
+            });
+
+            endRecordConversion(pageBuilder);
+        }
+        catch (RuntimeException e) { // TODO should use AddTimeRecordValidateException or the subclasses
+            log.warn(String.format("Skipped a record (%s).", e.getMessage()), e);
+        }
+    }
+
+    private void beginRecordConversion()
+    {
+    }
+
+    private void updateColumn(Column column, PageReader pageReader)
+    {
+        converters[column.getIndex()].update(pageReader);
+    }
+
+    private void endRecordConversion(PageBuilder pageBuilder)
+    {
+        for (ColumnConverter converter : converters) {
+            converter.convert(pageBuilder);
+        }
+
+        pageBuilder.addRecord();
+    }
+
+    public Schema toOutputSchema()
+    {
+        Schema.Builder schemaBuilder = new Schema.Builder();
+        for (ColumnConverter converter : converters) {
+            converter.addColumn(schemaBuilder);
+        }
+        return schemaBuilder.build();
+    }
+
+    static class AddTimeRecordValidateException
+            extends DataException
+    {
+        AddTimeRecordValidateException(Throwable cause)
+        {
+            super(cause);
+        }
+    }
+}