embulk-parser-joni_regexp 0.1.0

data/src/main/java/org/embulk/parser/joni_regexp/ColumnVisitorImpl.java

@@ -0,0 +1,167 @@
+package org.embulk.parser.joni_regexp;
+
+import com.google.common.base.Optional;
+import org.embulk.parser.joni_regexp.JoniRegexpParserPlugin.PluginTask;
+import org.embulk.parser.joni_regexp.JoniRegexpParserPlugin.TypecastColumnOption;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.msgpack.core.MessageTypeException;
+import org.msgpack.value.Value;
+
+public class ColumnVisitorImpl implements ColumnVisitor
+{
+    protected final PluginTask task;
+    protected final Schema schema;
+    protected final PageBuilder pageBuilder;
+    protected final TimestampParser[] timestampParsers;
+    protected final Boolean[] autoTypecasts;
+
+    protected Value value;
+
+    public ColumnVisitorImpl(PluginTask task, Schema schema, PageBuilder pageBuilder, TimestampParser[] timestampParsers)
+    {
+        this.task = task;
+        this.schema = schema;
+        this.pageBuilder = pageBuilder;
+        this.timestampParsers = timestampParsers.clone();
+        this.autoTypecasts = new Boolean[schema.size()];
+        buildAutoTypecasts();
+    }
+
+    private void buildAutoTypecasts()
+    {
+        for (Column column : schema.getColumns()) {
+            this.autoTypecasts[column.getIndex()] = task.getDefaultTypecast();
+        }
+
+        SchemaConfig schemaConfig = task.getColumns();
+
+        for (ColumnConfig columnConfig : schemaConfig.getColumns()) {
+            TypecastColumnOption columnOption = columnConfig.getOption().loadConfig(TypecastColumnOption.class);
+            Boolean autoTypecast = columnOption.getTypecast().or(task.getDefaultTypecast());
+            Column column = schema.lookupColumn(columnConfig.getName());
+            this.autoTypecasts[column.getIndex()] = autoTypecast;
+        }
+    }
+
+    public void setValue(Value value)
+    {
+        this.value = value;
+    }
+
+    @Override
+    public void booleanColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                boolean booleanValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asBoolean(value) : value.asBooleanValue().getBoolean();
+                pageBuilder.setBoolean(column, booleanValue);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Boolean", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void longColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                long longValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asLong(value) : value.asIntegerValue().toLong();
+                pageBuilder.setLong(column, longValue);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Long", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                double doubleValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asDouble(value) : value.asFloatValue().toDouble();
+                pageBuilder.setDouble(column, doubleValue);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed get \"%s\" as Double", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                String string = autoTypecasts[column.getIndex()] ? ColumnCaster.asString(value) : value.asStringValue().toString();
+                pageBuilder.setString(column, string);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as String", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                Timestamp timestamp = ColumnCaster.asTimestamp(value, timestampParsers[column.getIndex()]);
+                pageBuilder.setTimestamp(column, timestamp);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Timestamp", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                JsonParser parser = new JsonParser();
+                Value json = parser.parse(value.toString());
+                pageBuilder.setJson(column, json);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Json", value), e);
+            }
+        }
+    }
+
+    protected boolean isNil(Value v)
+    {
+        return v == null || v.isNilValue();
+    }
+}

data/src/main/java/org/embulk/parser/joni_regexp/JoniRegexpParserPlugin.java

@@ -0,0 +1,169 @@
+package org.embulk.parser.joni_regexp;
+
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.util.LineDecoder;
+import org.embulk.spi.util.Timestamps;
+import org.jcodings.specific.UTF8Encoding;
+import org.joni.Matcher;
+import org.joni.NameEntry;
+import org.joni.Option;
+import org.joni.Regex;
+import org.joni.Region;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+import org.slf4j.Logger;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Iterator;
+import java.util.Locale;
+
+public class JoniRegexpParserPlugin
+        implements ParserPlugin
+{
+    private static final Logger logger = Exec.getLogger(JoniRegexpParserPlugin.class);
+
+    public interface TypecastColumnOption
+            extends Task
+    {
+        @Config("typecast")
+        @ConfigDefault("null")
+        public Optional<Boolean> getTypecast();
+    }
+
+    public interface PluginTask
+            extends Task, LineDecoder.DecoderTask, TimestampParser.Task
+    {
+        @Config("columns")
+        SchemaConfig getColumns();
+
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
+
+        @Config("format")
+        String getFormat();
+
+        @Config("default_typecast")
+        @ConfigDefault("true")
+        Boolean getDefaultTypecast();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, ParserPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        Schema schema = task.getColumns().toSchema();
+
+        validateSchema(task, schema);
+
+        control.run(task.dump(), schema);
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema,
+            FileInput input, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        LineDecoder lineDecoder = new LineDecoder(input, task);
+        PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output);
+        TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getColumns());
+
+        ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, schema, pageBuilder, timestampParsers);
+
+        Regex regex = buildRegex(task);
+
+        while (input.nextFile()) {
+            while (true) {
+                String line = lineDecoder.poll();
+//                logger.debug(String.format(Locale.ENGLISH,"line = %s",line));
+                if (line == null) {
+                    break;
+                }
+                byte[] line_bytes = line.getBytes(StandardCharsets.UTF_8);
+                Matcher matcher = regex.matcher(line_bytes);
+                int result = matcher.search(0, line_bytes.length, Option.DEFAULT);
+
+                if (result != -1) {
+                    Region region = matcher.getEagerRegion();
+                    for (Iterator<NameEntry> entry = regex.namedBackrefIterator(); entry.hasNext(); ) {
+                        NameEntry e = entry.next();
+                        String name = captureName(e);
+
+                        int number = e.getBackRefs()[0];
+                        int begin = region.beg[number];
+                        int end = region.end[number];
+                        String strValue = new String(line_bytes, begin, end - begin, StandardCharsets.UTF_8);
+
+                        logger.debug(String.format(Locale.ENGLISH, "<%s> = %s", name, strValue));
+                        setValue(schema, visitor, name, strValue);
+                    }
+                    pageBuilder.addRecord();
+                }
+                else if (task.getStopOnInvalidRecord() == false) {
+                    logger.warn(String.format(Locale.ENGLISH, "skip unmatched line = %s", line));
+                }
+                else {
+                    throw new DataException(String.format("Invalid record at line %s", line));
+                }
+            }
+        }
+        pageBuilder.finish();
+    }
+
+    private void setValue(Schema schema, ColumnVisitorImpl visitor, String name, String strValue)
+    {
+        try {
+            Value value = ValueFactory.newString(strValue);
+            Column column = schema.lookupColumn(name);
+            visitor.setValue(value);
+            column.visit(visitor);
+        }
+        catch (Exception ex) {
+            throw new DataException(String.format(Locale.ENGLISH, "Set value failed. column = \"%s\" value = \"%s\", reason = \"%s\"", name, strValue, ex.getMessage()));
+        }
+    }
+
+    private Regex buildRegex(PluginTask task)
+    {
+        String format = task.getFormat();
+        byte[] pattern = format.getBytes(StandardCharsets.UTF_8);
+        // throw org.joni.exception.SyntaxException if regex is invalid.
+        return new Regex(pattern, 0, pattern.length, Option.NONE, UTF8Encoding.INSTANCE);
+    }
+
+    private String captureName(NameEntry e)
+    {
+        return new String(e.name, e.nameP, e.nameEnd - e.nameP);
+    }
+
+    private void validateSchema(PluginTask task, Schema schema)
+    {
+        Regex regex = buildRegex(task);
+        if (regex.numberOfNames() < 1) {
+            throw new ConfigException("The regex has no named capturing group");
+        }
+        for (Iterator<NameEntry> entry = regex.namedBackrefIterator(); entry.hasNext(); ) {
+            NameEntry e = entry.next();
+            String captureName = captureName(e);
+            schema.lookupColumn(captureName); // throw SchemaConfigException;
+        }
+    }
+}

data/src/main/java/org/embulk/parser/joni_regexp/JsonRecordValidateException.java

@@ -0,0 +1,22 @@
+package org.embulk.parser.joni_regexp;
+
+import org.embulk.spi.DataException;
+
+public class JsonRecordValidateException
+        extends DataException
+{
+    public JsonRecordValidateException(String message)
+    {
+        super(message);
+    }
+
+    public JsonRecordValidateException(String message, Throwable cause)
+    {
+        super(message, cause);
+    }
+
+    public JsonRecordValidateException(Throwable cause)
+    {
+        super(cause);
+    }
+}

data/src/main/java/org/embulk/parser/joni_regexp/cast/BooleanCast.java

@@ -0,0 +1,39 @@
+package org.embulk.parser.joni_regexp.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class BooleanCast
+{
+    private BooleanCast() {}
+
+    private static String buildErrorMessage(String as, boolean value)
+    {
+        return String.format("cannot cast boolean to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(boolean value) throws DataException
+    {
+        return value;
+    }
+
+    public static long asLong(boolean value) throws DataException
+    {
+        return value ? 1 : 0;
+    }
+
+    public static double asDouble(boolean value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("double", value));
+    }
+
+    public static String asString(boolean value) throws DataException
+    {
+        return value ? "true" : "false";
+    }
+
+    public static Timestamp asTimestamp(boolean value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("timestamp", value));
+    }
+}

data/src/main/java/org/embulk/parser/joni_regexp/cast/DoubleCast.java

@@ -0,0 +1,41 @@
+package org.embulk.parser.joni_regexp.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class DoubleCast
+{
+    private DoubleCast() {}
+
+    private static String buildErrorMessage(String as, double value)
+    {
+        return String.format("cannot cast double to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(double value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("boolean", value));
+    }
+
+    public static long asLong(double value) throws DataException
+    {
+        return (long) value;
+    }
+
+    public static double asDouble(double value) throws DataException
+    {
+        return value;
+    }
+
+    public static String asString(double value) throws DataException
+    {
+        return String.valueOf(value);
+    }
+
+    public static Timestamp asTimestamp(double value) throws DataException
+    {
+        long epochSecond = (long) value;
+        long nanoAdjustMent = (long) ((value - epochSecond) * 1000000000);
+        return Timestamp.ofEpochSecond(epochSecond, nanoAdjustMent);
+    }
+}

data/src/main/java/org/embulk/parser/joni_regexp/cast/JsonCast.java

@@ -0,0 +1,40 @@
+package org.embulk.parser.joni_regexp.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+
+public class JsonCast
+{
+    private JsonCast() {}
+
+    private static String buildErrorMessage(String as, Value value)
+    {
+        return String.format("cannot cast Json to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("boolean", value));
+    }
+
+    public static long asLong(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("long", value));
+    }
+
+    public static double asDouble(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("double", value));
+    }
+
+    public static String asString(Value value) throws DataException
+    {
+        return value.toString();
+    }
+
+    public static Timestamp asTimestamp(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("timestamp", value));
+    }
+}