embulk-parser-jsonline 0.2.2

data/src/main/java/org/embulk/parser/jsonl/ColumnCaster.java

@@ -0,0 +1,97 @@
+package org.embulk.parser.jsonl;
+
+import org.embulk.parser.jsonl.cast.BooleanCast;
+import org.embulk.parser.jsonl.cast.DoubleCast;
+import org.embulk.parser.jsonl.cast.JsonCast;
+import org.embulk.parser.jsonl.cast.LongCast;
+import org.embulk.parser.jsonl.cast.StringCast;
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.msgpack.value.Value;
+
+class ColumnCaster
+{
+    ColumnCaster() {}
+
+    public static boolean asBoolean(Value value) throws DataException
+    {
+        if (value.isBooleanValue()) {
+            return value.asBooleanValue().getBoolean();
+        }
+        else if (value.isIntegerValue()) {
+            return LongCast.asBoolean(value.asIntegerValue().asLong());
+        }
+        else if (value.isFloatValue()) {
+            return DoubleCast.asBoolean(value.asFloatValue().toDouble());
+        }
+        else if (value.isStringValue()) {
+            return StringCast.asBoolean(value.asStringValue().asString());
+        }
+        else {
+            return JsonCast.asBoolean(value);
+        }
+    }
+
+    public static long asLong(Value value) throws DataException
+    {
+        if (value.isBooleanValue()) {
+            return BooleanCast.asLong(value.asBooleanValue().getBoolean());
+        }
+        else if (value.isIntegerValue()) {
+            return value.asIntegerValue().asLong();
+        }
+        else if (value.isFloatValue()) {
+            return DoubleCast.asLong(value.asFloatValue().toDouble());
+        }
+        else if (value.isStringValue()) {
+            return StringCast.asLong(value.asStringValue().asString());
+        }
+        else {
+            return JsonCast.asLong(value);
+        }
+    }
+
+    public static double asDouble(Value value) throws DataException
+    {
+        if (value.isBooleanValue()) {
+            return BooleanCast.asDouble(value.asBooleanValue().getBoolean());
+        }
+        else if (value.isIntegerValue()) {
+            return LongCast.asDouble(value.asIntegerValue().asLong());
+        }
+        else if (value.isFloatValue()) {
+            return value.asFloatValue().toDouble();
+        }
+        else if (value.isStringValue()) {
+            return StringCast.asDouble(value.asStringValue().asString());
+        }
+        else {
+            return JsonCast.asDouble(value);
+        }
+    }
+
+    public static String asString(Value value) throws DataException
+    {
+        return value.toString();
+    }
+
+    public static Timestamp asTimestamp(Value value, TimestampParser parser) throws DataException
+    {
+        if (value.isBooleanValue()) {
+            return BooleanCast.asTimestamp(value.asBooleanValue().getBoolean());
+        }
+        else if (value.isIntegerValue()) {
+            return LongCast.asTimestamp(value.asIntegerValue().asLong());
+        }
+        else if (value.isFloatValue()) {
+            return DoubleCast.asTimestamp(value.asFloatValue().toDouble());
+        }
+        else if (value.isStringValue()) {
+            return StringCast.asTimestamp(value.asStringValue().asString(), parser);
+        }
+        else {
+            return JsonCast.asTimestamp(value);
+        }
+    }
+}

data/src/main/java/org/embulk/parser/jsonl/ColumnVisitorImpl.java

@@ -0,0 +1,164 @@
+package org.embulk.parser.jsonl;
+
+import com.google.common.base.Optional;
+import org.embulk.parser.jsonl.JsonlParserPlugin.PluginTask;
+import org.embulk.parser.jsonl.JsonlParserPlugin.TypecastColumnOption;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.msgpack.core.MessageTypeException;
+import org.msgpack.value.Value;
+
+public class ColumnVisitorImpl implements ColumnVisitor {
+    protected final PluginTask task;
+    protected final Schema schema;
+    protected final PageBuilder pageBuilder;
+    protected final TimestampParser[] timestampParsers;
+    protected final Boolean autoTypecasts[];
+
+    protected Value value;
+
+    public ColumnVisitorImpl(PluginTask task, Schema schema, PageBuilder pageBuilder, TimestampParser[] timestampParsers)
+    {
+        this.task = task;
+        this.schema = schema;
+        this.pageBuilder = pageBuilder;
+        this.timestampParsers = timestampParsers;
+        this.autoTypecasts = new Boolean[schema.size()];
+        buildAutoTypecasts();
+    }
+
+    private void buildAutoTypecasts()
+    {
+        for (Column column : schema.getColumns()) {
+            this.autoTypecasts[column.getIndex()] = task.getDefaultTypecast();
+        }
+
+        Optional<SchemaConfig> schemaConfig = task.getSchemaConfig();
+        if (schemaConfig.isPresent()) {
+            for (ColumnConfig columnConfig : schemaConfig.get().getColumns()) {
+                TypecastColumnOption columnOption = columnConfig.getOption().loadConfig(TypecastColumnOption.class);
+                Boolean autoTypecast = columnOption.getTypecast().or(task.getDefaultTypecast());
+                Column column = schema.lookupColumn(columnConfig.getName());
+                this.autoTypecasts[column.getIndex()] = autoTypecast;
+            }
+        }
+    }
+
+    public void setValue(Value value)
+    {
+        this.value = value;
+    }
+
+    @Override
+    public void booleanColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                boolean booleanValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asBoolean(value) : value.asBooleanValue().getBoolean();
+                pageBuilder.setBoolean(column, booleanValue);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Boolean", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void longColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                long longValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asLong(value) : value.asIntegerValue().toLong();
+                pageBuilder.setLong(column, longValue);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Long", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                double doubleValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asDouble(value) : value.asFloatValue().toDouble();
+                pageBuilder.setDouble(column, doubleValue);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed get \"%s\" as Double", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                String string = autoTypecasts[column.getIndex()] ? ColumnCaster.asString(value) : value.asStringValue().toString();
+                pageBuilder.setString(column, string);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as String", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                Timestamp timestamp = ColumnCaster.asTimestamp(value, timestampParsers[column.getIndex()]);
+                pageBuilder.setTimestamp(column, timestamp);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Timestamp", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                pageBuilder.setJson(column, value);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Json", value), e);
+            }
+        }
+    }
+
+    protected boolean isNil(Value v)
+    {
+        return v == null || v.isNilValue();
+    }
+}

data/src/main/java/org/embulk/parser/jsonl/JsonRecordValidateException.java

@@ -0,0 +1,22 @@
+package org.embulk.parser.jsonl;
+
+import org.embulk.spi.DataException;
+
+public class JsonRecordValidateException
+        extends DataException
+{
+    public JsonRecordValidateException(String message)
+    {
+        super(message);
+    }
+
+    public JsonRecordValidateException(String message, Throwable cause)
+    {
+        super(message, cause);
+    }
+
+    public JsonRecordValidateException(Throwable cause)
+    {
+        super(cause);
+    }
+}

data/src/main/java/org/embulk/parser/jsonl/JsonlParserPlugin.java

@@ -0,0 +1,225 @@
+package org.embulk.parser.jsonl;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.util.LineDecoder;
+import org.embulk.spi.util.Timestamps;
+import org.msgpack.value.Value;
+import org.slf4j.Logger;
+
+import java.util.Map;
+
+import static org.msgpack.value.ValueFactory.newString;
+
+public class JsonlParserPlugin
+        implements ParserPlugin
+{
+    @Deprecated
+    public interface JsonlColumnOption
+            extends Task
+    {
+        @Config("type")
+        @ConfigDefault("null")
+        Optional<Type> getType();
+    }
+
+    public interface TypecastColumnOption
+            extends Task
+    {
+        @Config("typecast")
+        @ConfigDefault("null")
+        public Optional<Boolean> getTypecast();
+    }
+
+    public interface PluginTask
+            extends Task, LineDecoder.DecoderTask, TimestampParser.Task
+    {
+        @Config("columns")
+        @ConfigDefault("null")
+        Optional<SchemaConfig> getSchemaConfig();
+
+        @Config("schema")
+        @ConfigDefault("null")
+        @Deprecated
+        Optional<SchemaConfig> getOldSchemaConfig();
+
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
+
+        @Config("default_typecast")
+        @ConfigDefault("true")
+        Boolean getDefaultTypecast();
+
+        @Config("column_options")
+        @ConfigDefault("{}")
+        @Deprecated
+        Map<String, JsonlColumnOption> getColumnOptions();
+    }
+
+    private final Logger log;
+
+    private String line = null;
+    private long lineNumber = 0;
+    private Map<String, Value> columnNameValues;
+
+    public JsonlParserPlugin()
+    {
+        this.log = Exec.getLogger(JsonlParserPlugin.class);
+    }
+
+    @Override
+    public void transaction(ConfigSource configSource, Control control)
+    {
+        PluginTask task = configSource.loadConfig(PluginTask.class);
+
+        if (! task.getColumnOptions().isEmpty()) {
+            log.warn("embulk-parser-jsonl: \"column_options\" option is deprecated, specify type directly to \"columns\" option with typecast: true (default: true).");
+        }
+
+        SchemaConfig schemaConfig = getSchemaConfig(task);
+        ImmutableList.Builder<Column> columns = ImmutableList.builder();
+        for (int i = 0; i < schemaConfig.getColumnCount(); i++) {
+            ColumnConfig columnConfig = schemaConfig.getColumn(i);
+            Type type = getType(task, columnConfig);
+            columns.add(new Column(i, columnConfig.getName(), type));
+        }
+        control.run(task.dump(), new Schema(columns.build()));
+    }
+
+    private static Type getType(PluginTask task, ColumnConfig columnConfig)
+    {
+        JsonlColumnOption columnOption = columnOptionOf(task.getColumnOptions(), columnConfig.getName());
+        return columnOption.getType().or(columnConfig.getType());
+    }
+
+    // this method is to keep the backward compatibility of 'schema' option.
+    private SchemaConfig getSchemaConfig(PluginTask task)
+    {
+        if (task.getOldSchemaConfig().isPresent()) {
+            log.warn("Please use 'columns' option instead of 'schema' because the 'schema' option is deprecated. The next version will stop 'schema' option support.");
+        }
+
+        if (task.getSchemaConfig().isPresent()) {
+            return task.getSchemaConfig().get();
+        }
+        else if (task.getOldSchemaConfig().isPresent()) {
+            return task.getOldSchemaConfig().get();
+        }
+        else {
+            throw new ConfigException("Attribute 'columns' is required but not set");
+        }
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        setColumnNameValues(schema);
+
+        final SchemaConfig schemaConfig = getSchemaConfig(task);
+        final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, schemaConfig);
+        final LineDecoder decoder = newLineDecoder(input, task);
+        final JsonParser jsonParser = newJsonParser();
+        final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
+
+        try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
+            ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, schema, pageBuilder, timestampParsers);
+
+            while (decoder.nextFile()) { // TODO this implementation should be improved with new JsonParser API on Embulk v0.8.3
+                lineNumber = 0;
+
+                while ((line = decoder.poll()) != null) {
+                    lineNumber++;
+
+                    try {
+                        Value value = jsonParser.parse(line);
+
+                        if (!value.isMapValue()) {
+                            throw new JsonRecordValidateException("Json string is not representing map value.");
+                        }
+
+                        final Map<Value, Value> record = value.asMapValue().map();
+                        for (Column column : schema.getColumns()) {
+                            Value v = record.get(getColumnNameValue(column));
+                            visitor.setValue(v);
+                            column.visit(visitor);
+                        }
+
+                        pageBuilder.addRecord();
+                    }
+                    catch (JsonRecordValidateException | JsonParseException e) {
+                        if (stopOnInvalidRecord) {
+                            throw new DataException(String.format("Invalid record at line %d: %s", lineNumber, line), e);
+                        }
+                        log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), line));
+                    }
+                }
+            }
+
+            pageBuilder.finish();
+        }
+    }
+
+    private void setColumnNameValues(Schema schema)
+    {
+        ImmutableMap.Builder<String, Value> builder = ImmutableMap.builder();
+        for (Column column : schema.getColumns()) {
+            String name = column.getName();
+            builder.put(name, newString(name));
+        }
+        columnNameValues = builder.build();
+    }
+
+    private Value getColumnNameValue(Column column)
+    {
+        return columnNameValues.get(column.getName());
+    }
+
+    public LineDecoder newLineDecoder(FileInput input, PluginTask task)
+    {
+        return new LineDecoder(input, task);
+    }
+
+    public JsonParser newJsonParser()
+    {
+        return new JsonParser();
+    }
+
+    private static JsonlColumnOption columnOptionOf(Map<String, JsonlColumnOption> columnOptions, String columnName)
+    {
+        return Optional.fromNullable(columnOptions.get(columnName)).or(
+                // default column option
+                new Supplier<JsonlColumnOption>()
+                {
+                    public JsonlColumnOption get()
+                    {
+                        return Exec.newConfigSource().loadConfig(JsonlColumnOption.class);
+                    }
+                });
+    }
+
+}

data/src/main/java/org/embulk/parser/jsonl/cast/BooleanCast.java

@@ -0,0 +1,39 @@
+package org.embulk.parser.jsonl.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class BooleanCast
+{
+    private BooleanCast() {}
+
+    private static String buildErrorMessage(String as, boolean value)
+    {
+        return String.format("cannot cast boolean to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(boolean value) throws DataException
+    {
+        return value;
+    }
+
+    public static long asLong(boolean value) throws DataException
+    {
+        return value ? 1 : 0;
+    }
+
+    public static double asDouble(boolean value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("double", value));
+    }
+
+    public static String asString(boolean value) throws DataException
+    {
+        return value ? "true" : "false";
+    }
+
+    public static Timestamp asTimestamp(boolean value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("timestamp", value));
+    }
+}

data/src/main/java/org/embulk/parser/jsonl/cast/DoubleCast.java

@@ -0,0 +1,41 @@
+package org.embulk.parser.jsonl.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class DoubleCast
+{
+    private DoubleCast() {}
+
+    private static String buildErrorMessage(String as, double value)
+    {
+        return String.format("cannot cast double to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(double value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("boolean", value));
+    }
+
+    public static long asLong(double value) throws DataException
+    {
+        return (long) value;
+    }
+
+    public static double asDouble(double value) throws DataException
+    {
+        return value;
+    }
+
+    public static String asString(double value) throws DataException
+    {
+        return String.valueOf(value);
+    }
+
+    public static Timestamp asTimestamp(double value) throws DataException
+    {
+        long epochSecond = (long) value;
+        long nanoAdjustMent = (long) ((value - epochSecond) * 1000000000);
+        return Timestamp.ofEpochSecond(epochSecond, nanoAdjustMent);
+    }
+}

data/src/main/java/org/embulk/parser/jsonl/cast/JsonCast.java

@@ -0,0 +1,40 @@
+package org.embulk.parser.jsonl.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+
+public class JsonCast
+{
+    private JsonCast() {}
+
+    private static String buildErrorMessage(String as, Value value)
+    {
+        return String.format("cannot cast Json to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("boolean", value));
+    }
+
+    public static long asLong(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("long", value));
+    }
+
+    public static double asDouble(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("double", value));
+    }
+
+    public static String asString(Value value) throws DataException
+    {
+        return value.toString();
+    }
+
+    public static Timestamp asTimestamp(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("timestamp", value));
+    }
+}

data/src/main/java/org/embulk/parser/jsonl/cast/LongCast.java

@@ -0,0 +1,47 @@
+package org.embulk.parser.jsonl.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class LongCast
+{
+    private LongCast() {}
+
+    private static String buildErrorMessage(String as, long value)
+    {
+        return String.format("cannot cast long to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(long value) throws DataException
+    {
+        if (value == 1) {
+            return true;
+        }
+        else if (value == 0) {
+            return false;
+        }
+        else {
+            throw new DataException(buildErrorMessage("boolean", value));
+        }
+    }
+
+    public static long asLong(long value) throws DataException
+    {
+        return value;
+    }
+
+    public static double asDouble(long value) throws DataException
+    {
+        return (double) value;
+    }
+
+    public static String asString(long value) throws DataException
+    {
+        return String.valueOf(value);
+    }
+
+    public static Timestamp asTimestamp(long value) throws DataException
+    {
+        return Timestamp.ofEpochSecond(value);
+    }
+}