embulk 0.8.39-java → 0.10.28-java

data/embulk-core/src/main/java/org/embulk/spi/unit/ToString.java

@@ -1,54 +0,0 @@
-package org.embulk.spi.unit;
-
-import com.google.common.base.Optional;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.node.NullNode;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonValue;
-
-public class ToString
-{
-    private final String string;
-
-    public ToString(String string)
-    {
-        this.string = string;
-    }
-
-    @JsonCreator
-    public ToString(Optional<JsonNode> option) throws JsonMappingException
-    {
-        JsonNode node = option.or(NullNode.getInstance());
-        if (node.isTextual()) {
-            this.string = node.textValue();
-        } else if (node.isValueNode()) {
-            this.string = node.toString();
-        } else {
-            throw new JsonMappingException(String.format("Arrays and objects are invalid: '%s'", node));
-        }
-    }
-
-    @Override
-    public boolean equals(Object obj)
-    {
-        if (!(obj instanceof ToString)) {
-            return false;
-        }
-        ToString o = (ToString) obj;
-        return string.equals(o.string);
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return string.hashCode();
-    }
-
-    @JsonValue
-    @Override
-    public String toString()
-    {
-        return string;
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/unit/ToStringMap.java

@@ -1,34 +0,0 @@
-package org.embulk.spi.unit;
-
-import java.util.Map;
-import java.util.HashMap;
-import java.util.Properties;
-import com.google.common.base.Function;
-import com.google.common.collect.Maps;
-import com.fasterxml.jackson.annotation.JsonCreator;
-
-public class ToStringMap
-        extends HashMap<String, String>
-{
-    @JsonCreator
-    public ToStringMap(Map<String, ToString> map)
-    {
-        super(Maps.transformValues(map, new Function<ToString, String>() {
-            public String apply(ToString value)
-            {
-                if (value == null) {
-                    return "null";
-                } else {
-                    return value.toString();
-                }
-            }
-        }));
-    }
-
-    public Properties toProperties()
-    {
-        Properties props = new Properties();
-        props.putAll(this);
-        return props;
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/CharsetSerDe.java

@@ -1,55 +0,0 @@
-package org.embulk.spi.util;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonSerializer;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
-import com.fasterxml.jackson.module.guice.ObjectMapperModule;
-
-public class CharsetSerDe
-{
-    public static void configure(ObjectMapperModule mapper)
-    {
-        SimpleModule module = new SimpleModule();
-        module.addSerializer(Charset.class, new CharsetSerializer());
-        module.addDeserializer(Charset.class, new CharsetDeserializer());
-        mapper.registerModule(module);
-    }
-
-    public static class CharsetSerializer
-            extends JsonSerializer<Charset>
-    {
-        @Override
-        public void serialize(Charset value, JsonGenerator jgen, SerializerProvider provider)
-                throws IOException
-        {
-            jgen.writeString(value.name());
-        }
-    }
-
-    public static class CharsetDeserializer
-            extends FromStringDeserializer<Charset>
-    {
-        public CharsetDeserializer()
-        {
-            super(Charset.class);
-        }
-
-        @Override
-        protected Charset _deserialize(String value, DeserializationContext context)
-                throws JsonMappingException
-        {
-            try {
-                return Charset.forName(value);
-            } catch (UnsupportedOperationException ex) {
-                // TODO include link to a document to the message for the list of supported time zones
-                throw new JsonMappingException(String.format("Unknown charset '%s'", value));
-            }
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/Decoders.java

@@ -1,81 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.List;
-import com.google.common.collect.ImmutableList;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.plugin.PluginType;
-import org.embulk.spi.ExecSession;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.DecoderPlugin;
-
-public abstract class Decoders
-{
-    private Decoders() { }
-
-    public static List<DecoderPlugin> newDecoderPlugins(ExecSession exec, List<ConfigSource> configs)
-    {
-        ImmutableList.Builder<DecoderPlugin> builder = ImmutableList.builder();
-        for (ConfigSource config : configs) {
-            builder.add(exec.newPlugin(DecoderPlugin.class, config.get(PluginType.class, "type")));
-        }
-        return builder.build();
-    }
-
-    public interface Control
-    {
-        public void run(List<TaskSource> taskSources);
-    }
-
-    public static void transaction(List<DecoderPlugin> plugins, List<ConfigSource> configs,
-            Decoders.Control control)
-    {
-        new RecursiveControl(plugins, configs, control).transaction();
-    }
-
-    public static FileInput open(List<DecoderPlugin> plugins, List<TaskSource> taskSources,
-            FileInput input)
-    {
-        FileInput in = input;
-        int pos = 0;
-        while (pos < plugins.size()) {
-            in = plugins.get(pos).open(taskSources.get(pos), in);
-            pos++;
-        }
-        return in;
-    }
-
-    private static class RecursiveControl
-    {
-        private final List<DecoderPlugin> plugins;
-        private final List<ConfigSource> configs;
-        private final Decoders.Control finalControl;
-        private final ImmutableList.Builder<TaskSource> taskSources;
-        private int pos;
-
-        RecursiveControl(List<DecoderPlugin> plugins, List<ConfigSource> configs,
-                Decoders.Control finalControl)
-        {
-            this.plugins = plugins;
-            this.configs = configs;
-            this.finalControl = finalControl;
-            this.taskSources = ImmutableList.builder();
-        }
-
-        public void transaction()
-        {
-            if (pos < plugins.size()) {
-                plugins.get(pos).transaction(configs.get(pos), new DecoderPlugin.Control() {
-                    public void run(TaskSource taskSource)
-                    {
-                        taskSources.add(taskSource);
-                        pos++;
-                        transaction();
-                    }
-                });
-            } else {
-                finalControl.run(taskSources.build());
-            }
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnNotFoundException.java

@@ -1,10 +0,0 @@
-package org.embulk.spi.util;
-
-public class DynamicColumnNotFoundException
-        extends RuntimeException
-{
-    public DynamicColumnNotFoundException(String message)
-    {
-        super(message);
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetter.java

@@ -1,21 +0,0 @@
-package org.embulk.spi.util;
-
-import org.embulk.spi.time.Timestamp;
-import org.msgpack.value.Value;
-
-public interface DynamicColumnSetter
-{
-    void setNull();
-
-    void set(boolean value);
-
-    void set(long value);
-
-    void set(double value);
-
-    void set(String value);
-
-    void set(Timestamp value);
-
-    void set(Value value);
-}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java

@@ -1,134 +0,0 @@
-package org.embulk.spi.util;
-
-import org.joda.time.DateTimeZone;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.type.Type;
-import org.embulk.spi.type.BooleanType;
-import org.embulk.spi.type.LongType;
-import org.embulk.spi.type.DoubleType;
-import org.embulk.spi.type.StringType;
-import org.embulk.spi.type.TimestampType;
-import org.embulk.spi.type.JsonType;
-import org.embulk.spi.util.dynamic.BooleanColumnSetter;
-import org.embulk.spi.util.dynamic.LongColumnSetter;
-import org.embulk.spi.util.dynamic.DoubleColumnSetter;
-import org.embulk.spi.util.dynamic.StringColumnSetter;
-import org.embulk.spi.util.dynamic.TimestampColumnSetter;
-import org.embulk.spi.util.dynamic.JsonColumnSetter;
-import org.embulk.spi.util.dynamic.DefaultValueSetter;
-import org.embulk.spi.util.dynamic.NullDefaultValueSetter;
-import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.spi.time.TimestampParser;
-import org.embulk.spi.time.TimestampFormat;
-import org.embulk.spi.Column;
-import org.embulk.spi.PageBuilder;
-import org.embulk.config.ConfigException;
-
-class DynamicColumnSetterFactory
-{
-    private final DefaultValueSetter defaultValue;
-    private final DynamicPageBuilder.BuilderTask task;
-    private final boolean useColumnForTimestampMetadata;
-
-    private DynamicColumnSetterFactory(
-            final DynamicPageBuilder.BuilderTask task,
-            final DefaultValueSetter defaultValue,
-            final boolean useColumnForTimestampMetadata)
-    {
-        this.defaultValue = defaultValue;
-        this.task = task;
-        this.useColumnForTimestampMetadata = useColumnForTimestampMetadata;
-    }
-
-    static DynamicColumnSetterFactory createWithTimestampMetadataFromBuilderTask(
-            final DynamicPageBuilder.BuilderTask task,
-            final DefaultValueSetter defaultValue)
-    {
-        return new DynamicColumnSetterFactory(task, defaultValue, false);
-    }
-
-    static DynamicColumnSetterFactory createWithTimestampMetadataFromColumn(
-            final DynamicPageBuilder.BuilderTask task,
-            final DefaultValueSetter defaultValue)
-    {
-        return new DynamicColumnSetterFactory(task, defaultValue, true);
-    }
-
-    public static DefaultValueSetter nullDefaultValue()
-    {
-        return new NullDefaultValueSetter();
-    }
-
-    public DynamicColumnSetter newColumnSetter(PageBuilder pageBuilder, Column column)
-    {
-        Type type = column.getType();
-        if (type instanceof BooleanType) {
-            return new BooleanColumnSetter(pageBuilder, column, defaultValue);
-        } else if (type instanceof LongType) {
-            return new LongColumnSetter(pageBuilder, column, defaultValue);
-        } else if (type instanceof DoubleType) {
-            return new DoubleColumnSetter(pageBuilder, column, defaultValue);
-        } else if (type instanceof StringType) {
-            TimestampFormatter formatter = new TimestampFormatter(
-                    getTimestampFormatForFormatter(column).getFormat(), getTimeZone(column));
-            return new StringColumnSetter(pageBuilder, column, defaultValue, formatter);
-        } else if (type instanceof TimestampType) {
-            // TODO use flexible time format like Ruby's Time.parse
-            final TimestampParser parser;
-            if (this.useColumnForTimestampMetadata) {
-                final TimestampType timestampType = (TimestampType) type;
-                parser = new TimestampParser(timestampType.getFormat(), getTimeZone(column));
-            }
-            else {
-                parser = new TimestampParser(
-                        getTimestampFormatForParser(column).getFormat(), getTimeZone(column));
-            }
-            return new TimestampColumnSetter(pageBuilder, column, defaultValue, parser);
-        } else if (type instanceof JsonType) {
-            TimestampFormatter formatter = new TimestampFormatter(
-                    getTimestampFormatForFormatter(column).getFormat(), getTimeZone(column));
-            return new JsonColumnSetter(pageBuilder, column, defaultValue, formatter);
-        }
-        throw new ConfigException("Unknown column type: "+type);
-    }
-
-    private TimestampFormat getTimestampFormatForFormatter(Column column)
-    {
-        DynamicPageBuilder.ColumnOption option = getColumnOption(column);
-        if (option != null) {
-            return option.getTimestampFormat();
-        } else {
-            return new TimestampFormat("%Y-%m-%d %H:%M:%S.%6N");
-        }
-    }
-
-    private TimestampFormat getTimestampFormatForParser(Column column)
-    {
-        DynamicPageBuilder.ColumnOption option = getColumnOption(column);
-        if (option != null) {
-            return option.getTimestampFormat();
-        } else {
-            return new TimestampFormat("%Y-%m-%d %H:%M:%S.%N");
-        }
-    }
-
-    private DateTimeZone getTimeZone(Column column)
-    {
-        DynamicPageBuilder.ColumnOption option = getColumnOption(column);
-        if (option != null) {
-            return option.getTimeZone().or(task.getDefaultTimeZone());
-        } else {
-            return task.getDefaultTimeZone();
-        }
-    }
-
-    private DynamicPageBuilder.ColumnOption getColumnOption(Column column)
-    {
-        ConfigSource option = task.getColumnOptions().get(column.getName());
-        if (option != null) {
-            return option.loadConfig(DynamicPageBuilder.ColumnOption.class);
-        } else {
-            return null;
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java

@@ -1,200 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.List;
-import java.util.Map;
-import com.google.common.base.Optional;
-import org.joda.time.DateTimeZone;
-import org.jruby.embed.ScriptingContainer;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.spi.Schema;
-import org.embulk.spi.Column;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.spi.time.TimestampParser;
-import org.embulk.spi.time.TimestampFormat;
-import org.embulk.spi.util.dynamic.SkipColumnSetter;
-
-public class DynamicPageBuilder
-        implements AutoCloseable
-{
-    private final PageBuilder pageBuilder;
-    private final Schema schema;
-    private final DynamicColumnSetter[] setters;
-    private final Map<String, DynamicColumnSetter> columnLookup;
-
-    public static interface BuilderTask
-            extends Task
-    {
-        @Config("default_timezone")
-        @ConfigDefault("\"UTC\"")
-        public DateTimeZone getDefaultTimeZone();
-
-        @Config("column_options")
-        @ConfigDefault("{}")
-        public Map<String, ConfigSource> getColumnOptions();
-
-        // required by TimestampFormatter
-        @ConfigInject
-        @Deprecated
-        public ScriptingContainer getJRuby();
-    }
-
-    public static interface ColumnOption
-            extends Task
-    {
-        // DynamicPageBuilder is used for inputs, then datetime parsing.
-        // Ruby's strptime does not accept numeric prefixes in specifiers such as "%6N".
-        @Config("timestamp_format")
-        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%N\"")
-        public TimestampFormat getTimestampFormat();
-
-        @Config("timezone")
-        @ConfigDefault("null")
-        public Optional<DateTimeZone> getTimeZone();
-    }
-
-    private DynamicPageBuilder(
-            final DynamicColumnSetterFactory factory,
-            final BufferAllocator allocator,
-            final Schema schema,
-            final PageOutput output)
-    {
-        this.pageBuilder = new PageBuilder(allocator, schema, output);
-        this.schema = schema;
-        ImmutableList.Builder<DynamicColumnSetter> setters = ImmutableList.builder();
-        ImmutableMap.Builder<String, DynamicColumnSetter> lookup = ImmutableMap.builder();
-        for (Column c : schema.getColumns()) {
-            DynamicColumnSetter setter = factory.newColumnSetter(pageBuilder, c);
-            setters.add(setter);
-            lookup.put(c.getName(), setter);
-        }
-        this.setters = setters.build().toArray(new DynamicColumnSetter[0]);
-        this.columnLookup = lookup.build();
-    }
-
-    @Deprecated  // The static creator method "createWithTimestampMetadataFromBuilderTask" is preferred.
-    public DynamicPageBuilder(
-            BuilderTask task,
-            BufferAllocator allocator,
-            Schema schema,
-            PageOutput output)
-    {
-        this(DynamicColumnSetterFactory.createWithTimestampMetadataFromBuilderTask(
-                 task, DynamicColumnSetterFactory.nullDefaultValue()),  // TODO configurable default value
-             allocator,
-             schema,
-             output);
-    }
-
-    public static DynamicPageBuilder createWithTimestampMetadataFromBuilderTask(
-            BuilderTask task,
-            BufferAllocator allocator,
-            Schema schema,
-            PageOutput output)
-    {
-        // TODO configurable default value
-        DynamicColumnSetterFactory factory = DynamicColumnSetterFactory.createWithTimestampMetadataFromBuilderTask(
-                task, DynamicColumnSetterFactory.nullDefaultValue());
-        return new DynamicPageBuilder(factory, allocator, schema, output);
-    }
-
-    public static DynamicPageBuilder createWithTimestampMetadataFromColumn(
-            BuilderTask task,
-            BufferAllocator allocator,
-            Schema schema,
-            PageOutput output)
-    {
-        // TODO configurable default value
-        DynamicColumnSetterFactory factory = DynamicColumnSetterFactory.createWithTimestampMetadataFromColumn(
-                task, DynamicColumnSetterFactory.nullDefaultValue());
-        return new DynamicPageBuilder(factory, allocator, schema, output);
-    }
-
-    public List<Column> getColumns()
-    {
-        return schema.getColumns();
-    }
-
-    public DynamicColumnSetter column(Column c)
-    {
-        return setters[c.getIndex()];
-    }
-
-    public DynamicColumnSetter column(int index)
-    {
-        if (index < 0 || setters.length <= index) {
-            throw new DynamicColumnNotFoundException("Column index '"+index+"' is not exist");
-        }
-        return setters[index];
-    }
-
-    public DynamicColumnSetter lookupColumn(String columnName)
-    {
-        DynamicColumnSetter setter = columnLookup.get(columnName);
-        if (setter == null) {
-            throw new DynamicColumnNotFoundException("Column '"+columnName+"' is not exist");
-        }
-        return setter;
-    }
-
-    public DynamicColumnSetter columnOrSkip(int index)
-    {
-        if (index < 0 || setters.length <= index) {
-            return SkipColumnSetter.get();
-        }
-        return setters[index];
-    }
-
-    public DynamicColumnSetter columnOrSkip(String columnName)
-    {
-        DynamicColumnSetter setter = columnLookup.get(columnName);
-        if (setter == null) {
-            return SkipColumnSetter.get();
-        }
-        return setter;
-    }
-
-    // for jruby
-    protected DynamicColumnSetter columnOrNull(int index)
-    {
-        if (index < 0 || setters.length <= index) {
-            return null;
-        }
-        return setters[index];
-    }
-
-    // for jruby
-    protected DynamicColumnSetter columnOrNull(String columnName)
-    {
-        return columnLookup.get(columnName);
-    }
-
-    public void addRecord()
-    {
-        pageBuilder.addRecord();
-    }
-
-    public void flush()
-    {
-        pageBuilder.flush();
-    }
-
-    public void finish()
-    {
-        pageBuilder.finish();
-    }
-
-    @Override
-    public void close()
-    {
-        pageBuilder.close();
-    }
-}