embulk 0.7.11-java → 0.8.0-java

data/embulk-core/src/main/java/org/embulk/spi/Column.java

@@ -9,6 +9,7 @@ import org.embulk.spi.type.DoubleType;
 import org.embulk.spi.type.LongType;
 import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.type.StringType;
+import org.embulk.spi.type.JsonType;
 
 public class Column
 {
@@ -57,6 +58,8 @@ public class Column
             visitor.stringColumn(this);
         } else if (type instanceof TimestampType) {
             visitor.timestampColumn(this);
+        } else if (type instanceof JsonType) {
+            visitor.jsonColumn(this);
         } else {
             assert(false);
         }

data/embulk-core/src/main/java/org/embulk/spi/ColumnVisitor.java

@@ -11,4 +11,6 @@ public interface ColumnVisitor
     void stringColumn(Column column);
 
     void timestampColumn(Column column);
+
+    void jsonColumn(Column column);
 }

data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java

@@ -47,6 +47,7 @@ public class ExecSession
     public static class Builder
     {
         private final Injector injector;
+        private ILoggerFactory loggerFactory;
         private Timestamp transactionTime;
 
         public Builder(Injector injector)
@@ -60,6 +61,12 @@ public class ExecSession
             return this;
         }
 
+        public Builder setLoggerFactory(ILoggerFactory loggerFactory)
+        {
+            this.loggerFactory = loggerFactory;
+            return this;
+        }
+
         public Builder setTransactionTime(Timestamp timestamp)
         {
             this.transactionTime = timestamp;
@@ -71,7 +78,7 @@ public class ExecSession
             if (transactionTime == null) {
                 transactionTime = Timestamp.ofEpochMilli(System.currentTimeMillis());  // TODO get nanoseconds for default
             }
-            return new ExecSession(injector, transactionTime);
+            return new ExecSession(injector, transactionTime, Optional.fromNullable(loggerFactory));
         }
     }
 
@@ -86,14 +93,14 @@ public class ExecSession
         this(injector,
                 configSource.loadConfig(SessionTask.class).getTransactionTime().or(
                     Timestamp.ofEpochMilli(System.currentTimeMillis())
-                )
-            );  // TODO get nanoseconds for default
+                    ), // TODO get nanoseconds for default
+                null);
     }
 
-    private ExecSession(Injector injector, Timestamp transactionTime)
+    private ExecSession(Injector injector, Timestamp transactionTime, Optional<ILoggerFactory> loggerFactory)
     {
         this.injector = injector;
-        this.loggerFactory = injector.getInstance(ILoggerFactory.class);
+        this.loggerFactory = loggerFactory.or(injector.getInstance(ILoggerFactory.class));
         this.modelManager = injector.getInstance(ModelManager.class);
         this.pluginManager = injector.getInstance(PluginManager.class);
         this.bufferAllocator = injector.getInstance(BufferAllocator.class);

data/embulk-core/src/main/java/org/embulk/spi/Page.java

@@ -1,11 +1,13 @@
 package org.embulk.spi;
 
 import java.util.List;
+import org.msgpack.value.ImmutableValue;
 
 public class Page
 {
     private final Buffer buffer;
     private List<String> stringReferences;
+    private List<ImmutableValue> valueReferences;
 
     protected Page(Buffer buffer)
     {
@@ -28,17 +30,34 @@ public class Page
         return this;
     }
 
+    public Page setValueReferences(List<ImmutableValue> values)
+    {
+        this.valueReferences = values;
+        return this;
+    }
+
     public List<String> getStringReferences()
     {
         // TODO used by mapreduce executor
         return stringReferences;
     }
 
+    public List<ImmutableValue> getValueReferences()
+    {
+        // TODO used by mapreduce executor
+        return valueReferences;
+    }
+
     public String getStringReference(int index)
     {
         return stringReferences.get(index);
     }
 
+    public ImmutableValue getValueReference(int index)
+    {
+        return valueReferences.get(index);
+    }
+
     public void release()
     {
         buffer.release();

data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java

@@ -11,6 +11,8 @@ import com.google.common.collect.BiMap;
 import com.google.common.collect.HashBiMap;
 import io.airlift.slice.Slice;
 import io.airlift.slice.Slices;
+import org.msgpack.value.Value;
+import org.msgpack.value.ImmutableValue;
 import org.embulk.spi.time.Timestamp;
 
 public class PageBuilder
@@ -29,7 +31,8 @@ public class PageBuilder
     private int position;
     private final byte[] nullBitSet;
     private final BiMap<String, Integer> stringReferences = HashBiMap.create();
-    private int stringReferenceSize;
+    private List<ImmutableValue> valueReferences = new ArrayList<>();
+    private int referenceSize;
     private int nextVariableLengthDataOffset;
 
     public PageBuilder(BufferAllocator allocator, Schema schema, PageOutput output)
@@ -52,7 +55,8 @@ public class PageBuilder
         this.count = 0;
         this.position = PageFormat.PAGE_HEADER_SIZE;
         this.stringReferences.clear();
-        this.stringReferenceSize = 0;
+        this.valueReferences = new ArrayList<>();
+        this.referenceSize = 0;
     }
 
     public Schema getSchema()
@@ -126,11 +130,26 @@ public class PageBuilder
             int index = stringReferences.size();
             stringReferences.put(value, index);
             bufferSlice.setInt(getOffset(columnIndex), index);
-            stringReferenceSize += value.length() * 2 + 4;  // assuming size of char = size of byte * 2 + length
+            referenceSize += value.length() * 2 + 4;  // assuming size of char = size of byte * 2 + length
         }
         clearNull(columnIndex);
     }
 
+    public void setJson(Column column, Value value)
+    {
+        // TODO check type?
+        setJson(column.getIndex(), value);
+    }
+
+    public void setJson(int columnIndex, Value value)
+    {
+        int index = valueReferences.size();
+        valueReferences.add(value.immutableValue());
+        bufferSlice.setInt(getOffset(columnIndex), index);
+        referenceSize += 256;  // TODO how to estimate size of the value?
+        clearNull(columnIndex);
+    }
+
     public void setTimestamp(Column column, Timestamp value)
     {
         // TODO check type?
@@ -189,7 +208,7 @@ public class PageBuilder
         Arrays.fill(nullBitSet, (byte) -1);
 
         // flush if next record will not fit in this buffer
-        if (buffer.capacity() < position + nextVariableLengthDataOffset + stringReferenceSize) {
+        if (buffer.capacity() < position + nextVariableLengthDataOffset + referenceSize) {
             flush();
         }
     }
@@ -202,7 +221,9 @@ public class PageBuilder
             buffer.limit(position);
 
             // flush page
-            Page page = Page.wrap(buffer).setStringReferences(getSortedStringReferences());
+            Page page = Page.wrap(buffer)
+                .setStringReferences(getSortedStringReferences())
+                .setValueReferences(valueReferences);
             buffer = null;
             bufferSlice = null;
             output.add(page);

data/embulk-core/src/main/java/org/embulk/spi/PageReader.java

@@ -2,6 +2,7 @@ package org.embulk.spi;
 
 import io.airlift.slice.Slice;
 import io.airlift.slice.Slices;
+import org.msgpack.value.Value;
 import org.embulk.spi.time.Timestamp;
 
 public class PageReader
@@ -124,6 +125,18 @@ public class PageReader
         return Timestamp.ofEpochSecond(sec, nsec);
     }
 
+    public Value getJson(Column column)
+    {
+        // TODO check type?
+        return getJson(column.getIndex());
+    }
+
+    public Value getJson(int columnIndex)
+    {
+        int index = pageSlice.getInt(getOffset(columnIndex));
+        return page.getValueReference(index);
+    }
+
     private int getOffset(int columnIndex)
     {
         return position + columnOffsets[columnIndex];

data/embulk-core/src/main/java/org/embulk/spi/json/JsonParseException.java

@@ -0,0 +1,17 @@
+package org.embulk.spi.json;
+
+import org.embulk.spi.DataException;
+
+public class JsonParseException
+        extends DataException
+{
+    public JsonParseException(String message)
+    {
+        super(message);
+    }
+
+    public JsonParseException(String message, Throwable cause)
+    {
+        super(message, cause);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/json/JsonParser.java

@@ -0,0 +1,125 @@
+package org.embulk.spi.json;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.io.IOException;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser.Feature;
+import com.fasterxml.jackson.core.JsonToken;
+
+public class JsonParser
+{
+    private final JsonFactory factory;
+
+    public JsonParser()
+    {
+        this.factory = new JsonFactory();
+        factory.enable(Feature.ALLOW_UNQUOTED_CONTROL_CHARS);
+    }
+
+    public Value parse(String json)
+    {
+        return new ParseContext(json).parse();
+    }
+
+    private class ParseContext
+    {
+        private final String json;
+        private final com.fasterxml.jackson.core.JsonParser parser;
+
+        public ParseContext(String json)
+        {
+            this.json = json;
+            try {
+                this.parser = factory.createParser(json);
+            }
+            catch (Exception ex) {
+                throw new JsonParseException("Failed to parse a JSON string: "+sampleJsonString(json), ex);
+            }
+        }
+
+        public Value parse()
+        {
+            try {
+                JsonToken token = parser.nextToken();
+                return jsonTokenToValue(token);
+            }
+            catch (JsonParseException ex) {
+                throw ex;
+            }
+            catch (Exception ex) {
+                throw new JsonParseException("Failed to parse a JSON string: "+sampleJsonString(json), ex);
+            }
+        }
+
+        private Value jsonTokenToValue(JsonToken token)
+            throws IOException
+        {
+            switch(token) {
+            case VALUE_NULL:
+                return ValueFactory.newNil();
+            case VALUE_TRUE:
+                return ValueFactory.newBoolean(true);
+            case VALUE_FALSE:
+                return ValueFactory.newBoolean(false);
+            case VALUE_NUMBER_FLOAT:
+                return ValueFactory.newFloat(parser.getDoubleValue());
+            case VALUE_NUMBER_INT:
+                try {
+                    return ValueFactory.newInteger(parser.getLongValue());
+                }
+                catch (JsonParseException ex) {
+                    return ValueFactory.newInteger(parser.getBigIntegerValue());
+                }
+            case VALUE_STRING:
+                return ValueFactory.newString(parser.getText());
+            case START_ARRAY: {
+                List<Value> list = new ArrayList<>();
+                while (true) {
+                    token = parser.nextToken();
+                    if(token == JsonToken.END_ARRAY) {
+                        return ValueFactory.newArray(list);
+                    }
+                    list.add(jsonTokenToValue(token));
+                }
+            }
+            case START_OBJECT:
+                Map<Value, Value> map = new HashMap<>();
+                while (true) {
+                    token = parser.nextToken();
+                    if (token == JsonToken.END_OBJECT) {
+                        return ValueFactory.newMap(map);
+                    }
+                    String key = parser.getCurrentName();
+                    if (key == null) {
+                        throw new JsonParseException("Unexpected token "+token+" at "+parser.getTokenLocation());
+                    }
+                    token = parser.nextToken();
+                    Value value = jsonTokenToValue(token);
+                    map.put(ValueFactory.newString(key), value);
+                }
+            case VALUE_EMBEDDED_OBJECT:
+            case FIELD_NAME:
+            case END_ARRAY:
+            case END_OBJECT:
+            case NOT_AVAILABLE:
+            default:
+                throw new JsonParseException("Unexpected token "+token+" at "+parser.getTokenLocation());
+            }
+        }
+    }
+
+    private static String sampleJsonString(String json)
+    {
+        if (json.length() < 100) {
+            return json;
+        }
+        else {
+            return json.substring(0, 97) + "...";
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/json/RubyValueApi.java

@@ -0,0 +1,55 @@
+package org.embulk.spi.json;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import com.google.common.base.Throwables;
+import org.msgpack.core.MessagePack;
+import org.msgpack.core.MessageBufferPacker;
+import org.msgpack.core.buffer.MessageBuffer;
+import org.msgpack.value.Value;
+import org.jruby.Ruby;
+import org.jruby.RubyString;
+import org.jruby.util.ByteList;
+import org.jcodings.specific.ASCIIEncoding;
+
+public class RubyValueApi
+{
+    public static Value fromMessagePack(RubyString content)
+    {
+        ByteList list = content.getByteList();
+        try {
+            return MessagePack.newDefaultUnpacker(list.unsafeBytes(), list.begin(), list.length()).unpackValue();
+        }
+        catch (IOException ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    private static class OpenByteArrayOutputStream
+            extends ByteArrayOutputStream
+    {
+        public byte[] getBuffer()
+        {
+            return buf;
+        }
+
+        public int getCount()
+        {
+            return count;
+        }
+    }
+
+    public static RubyString toMessagePack(Ruby runtime, Value value)
+    {
+        try {
+            MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();
+            packer.packValue(value);
+            MessageBuffer mb = packer.toMessageBuffer();
+            ByteList list = new ByteList(mb.array(), mb.arrayOffset(), mb.size(), ASCIIEncoding.INSTANCE, false);
+            return RubyString.newString(runtime, list);
+        }
+        catch (IOException ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/JsonType.java

@@ -0,0 +1,14 @@
+package org.embulk.spi.type;
+
+import org.msgpack.value.Value;
+
+public class JsonType
+        extends AbstractType
+{
+    static final JsonType JSON = new JsonType();
+
+    private JsonType()
+    {
+        super("json", Value.class, 4);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java

@@ -20,6 +20,7 @@ public class TypeDeserializer
         builder.put(DoubleType.DOUBLE.getName(), DoubleType.DOUBLE);
         builder.put(StringType.STRING.getName(), StringType.STRING);
         builder.put(TimestampType.TIMESTAMP.getName(), TimestampType.TIMESTAMP);
+        builder.put(JsonType.JSON.getName(), JsonType.JSON);
         stringToTypeMap = builder.build();
     }