embulk-parser-msgpack 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 90cbac3b997c6ed3d27a9f6bb4f7f534f8e4a1b8
-  data.tar.gz: 2d9be2d25cdb4fc7ab221c468a790d303ac99df2
+  metadata.gz: 5b4c775c2942e56f3df0d9a6af992220c4fa1d4a
+  data.tar.gz: ef29ab00cffc8e5f5df887586cfd83e0bfafd955
 SHA512:
-  metadata.gz: f8268bf872a5dcb689f595f223102f4b761cd0e55e9678b9af47ba63c434cdedc2c37396efdb9a88442170f89d4d90422d09e4fda68e1374928baec68ac5af02
-  data.tar.gz: 57ddd8e8b72d2a35c375c165aeb3c20fac5f554022fa8f710e405c1c81b5bd554dc0fb9a74aaf429bdf0f9c27ead223aba2cc967c2dc598a46e315d03f5e34f4
+  metadata.gz: e52e2c9ffdfaa491eb1298e31bbbe55661c06ef0b3b41f1230713dfaa8bb1c46984ffa907c4ae9f445229eeb91fb9ee17af10e43e0844d12bda2598130a4a5d3
+  data.tar.gz: aa5690f19cc16469e8935e8599e61eb68602ffd1512152ec78be437bdb70a7f3018f020d8aaa5c14eefe6d720b09d7c2d3da93528c430752e42439edf69ef7d0

data/.travis.yml

@@ -0,0 +1,6 @@
+language: java
+jdk:
+  - oraclejdk7
+script:
+  - ./gradlew gem
+  - ./gradlew --info check jacocoTestReport

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.2.2 - 2016-11-03
+
+* Make 'columns' config optional and enable schemaless data parsing [#5, #6, #7]
+* Upgraded Embulk version to v0.8.14 [#3]
+
 Release 0.2.1 - 2016-02-24
 
 * Upgraded msgpack-java version to v0.8.3

data/README.md

@@ -11,7 +11,7 @@ Parses files encoded in MessagePack.
 
 - **row_encoding**: type of a row. "array" or "map" (enum, default: map)
 - **file_encoding**: if a file includes a big array, set "array". Otherwise, if a file includes sequence of rows, set "sequence" (enum, default: sequence)
-- **columns**: description (schema, required)
+- **columns**: description (schema, default: a single Json typed column)
 
 ## Example
 

data/build.gradle

@@ -3,6 +3,8 @@ plugins {
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
     id "checkstyle"
+    id "findbugs"
+    id "jacoco"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -14,15 +16,17 @@ configurations {
     provided
 }
 
-version = "0.2.1"
+version = "0.2.2"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.3"
-    provided "org.embulk:embulk-core:0.8.3"
+    compile  "org.embulk:embulk-core:0.8.14"
+    provided "org.embulk:embulk-core:0.8.14"
     testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.14:tests"
+    testCompile "org.embulk:embulk-standards:0.8.14"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -49,6 +53,16 @@ task checkstyle(type: Checkstyle) {
     source = sourceSets.main.allJava + sourceSets.test.allJava
 }
 
+tasks.withType(FindBugs) {
+    reports {
+        xml.enabled = false
+        html.enabled = true
+    }
+}
+findbugs {
+    ignoreFailures = true
+}
+
 task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
     jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
     script "${project.name}.gemspec"

data/src/main/java/org/embulk/parser/msgpack/MsgpackParserPlugin.java

@@ -6,16 +6,23 @@ import java.util.TreeMap;
 import java.util.Comparator;
 import java.io.IOException;
 import java.io.EOFException;
+
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.collect.Lists;
+import org.embulk.spi.Exec;
+import org.embulk.spi.type.Types;
 import org.msgpack.core.MessagePack;
 import org.msgpack.core.MessageFormat;
 import org.msgpack.core.MessageUnpacker;
 import org.msgpack.core.MessageInsufficientBufferException;
 import org.msgpack.core.buffer.MessageBuffer;
 import org.msgpack.core.buffer.MessageBufferInput;
+import org.msgpack.value.Value;
 import org.msgpack.value.ValueType;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigException;
@@ -58,6 +65,9 @@ import org.embulk.spi.util.dynamic.JsonColumnSetter;
 import org.embulk.spi.util.dynamic.DefaultValueSetter;
 import org.embulk.spi.util.dynamic.NullDefaultValueSetter;
 
+import static org.embulk.spi.Exec.newConfigSource;
+import static org.embulk.spi.type.Types.*;
+
 public class MsgpackParserPlugin
         implements ParserPlugin
 {
@@ -73,10 +83,14 @@ public class MsgpackParserPlugin
         public RowEncoding getRowEncoding();
 
         @Config("columns")
-        public SchemaConfig getSchemaConfig();
+        @ConfigDefault("null")
+        public Optional<SchemaConfig> getSchemaConfig();
 
         @ConfigInject
         public BufferAllocator getBufferAllocator();
+
+        public void setSchemafulMode(boolean v);
+        public boolean getSchemafulMode();
     }
 
     public static enum FileEncoding
@@ -195,7 +209,30 @@ public class MsgpackParserPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        control.run(task.dump(), task.getSchemaConfig().toSchema());
+        if (!task.getSchemaConfig().isPresent()) {
+            // If columns: is not set, the parser behaves as non-schemaful mode. It doesn't care of row encoding.
+            if (config.has("row_encoding")) {
+                throw new ConfigException("Setting row_encoding: is invalid if columns: is not set.");
+            }
+            task.setSchemafulMode(false);
+        }
+        else {
+            task.setSchemafulMode(true);
+        }
+
+        control.run(task.dump(), getSchemaConfig(task).toSchema());
+    }
+
+    @VisibleForTesting
+    SchemaConfig getSchemaConfig(PluginTask task)
+    {
+        Optional<SchemaConfig> schemaConfig = task.getSchemaConfig();
+        if (schemaConfig.isPresent()) {
+            return schemaConfig.get();
+        }
+        else {
+            return new SchemaConfig(ImmutableList.of(new ColumnConfig("record", JSON, newConfigSource())));
+        }
     }
 
     @Override
@@ -204,41 +241,75 @@ public class MsgpackParserPlugin
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
-        RowEncoding rowEncoding = task.getRowEncoding();
+        boolean schemafulMode = task.getSchemafulMode();
         FileEncoding fileEncoding = task.getFileEncoding();
 
         try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(new FileInputMessageBufferInput(input));
                 PageBuilder pageBuilder = new PageBuilder(task.getBufferAllocator(), schema, output)) {
 
-            TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig());
-            Map<Column, DynamicColumnSetter> setters = newColumnSetters(pageBuilder,
-                    task.getSchemaConfig(), timestampParsers, taskSource.loadTask(PluginTaskFormatter.class));
-
-            RowReader reader;
-            switch (rowEncoding) {
-            case ARRAY:
-                reader = new ArrayRowReader(setters);
-                break;
-            case MAP:
-                reader = new MapRowReader(setters);
-                break;
-            default:
-                throw new IllegalArgumentException("Unexpected row encoding");
-            }
+            if (schemafulMode) {
+                RowEncoding rowEncoding = task.getRowEncoding();
+                TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, getSchemaConfig(task));
+                Map<Column, DynamicColumnSetter> setters = newColumnSetters(pageBuilder,
+                        getSchemaConfig(task), timestampParsers, taskSource.loadTask(PluginTaskFormatter.class));
 
-            while (input.nextFile()) {
-                switch (fileEncoding) {
-                case SEQUENCE:
-                    // do nothing
-                    break;
+                RowReader reader;
+                switch (rowEncoding) {
                 case ARRAY:
-                    // skip array header to convert array to sequence
-                    unpacker.unpackArrayHeader();
+                    reader = new ArrayRowReader(setters);
+                    break;
+                case MAP:
+                    reader = new MapRowReader(setters);
                     break;
+                default:
+                    throw new IllegalArgumentException("Unexpected row encoding");
+                }
+
+                while (input.nextFile()) {
+                    switch (fileEncoding) {
+                    case SEQUENCE:
+                        // do nothing
+                        break;
+                    case ARRAY:
+                        // skip array header to convert array to sequence
+                        unpacker.unpackArrayHeader();
+                        break;
+                    }
+
+                    while (reader.next(unpacker)) {
+                        pageBuilder.addRecord();
+                    }
                 }
+            }
+            else {
+                // If non-schemaful mode, setters is not created.
+                while (input.nextFile()) {
+                    switch (fileEncoding) {
+                    case SEQUENCE:
+                        // do nothing
+                        break;
+                    case ARRAY:
+                        // skip array header to convert array to sequence
+                        unpacker.unpackArrayHeader();
+                        break;
+                    }
 
-                while (reader.next(unpacker)) {
-                    pageBuilder.addRecord();
+                    while (true) {
+                        Value v;
+                        try {
+                            v = unpacker.unpackValue();
+                            if (v == null) {
+                                break;
+                            }
+                        }
+                        catch (MessageInsufficientBufferException e) {
+                            break;
+                        }
+
+                        // The unpacked Value object is set to a page as a Json column value.
+                        pageBuilder.setJson(0, v);
+                        pageBuilder.addRecord();
+                    }
                 }
             }
 
@@ -264,29 +335,35 @@ public class MsgpackParserPlugin
             if (type instanceof BooleanType) {
                 setter = new BooleanColumnSetter(pageBuilder, column, defaultValue);
 
-            } else if (type instanceof LongType) {
+            }
+            else if (type instanceof LongType) {
                 setter = new LongColumnSetter(pageBuilder, column, defaultValue);
 
-            } else if (type instanceof DoubleType) {
+            }
+            else if (type instanceof DoubleType) {
                 setter = new DoubleColumnSetter(pageBuilder, column, defaultValue);
 
-            } else if (type instanceof StringType) {
+            }
+            else if (type instanceof StringType) {
                 TimestampFormatter formatter = new TimestampFormatter(formatterTask,
                         Optional.of(c.getOption().loadConfig(TimestampColumnOption.class)));
                 setter = new StringColumnSetter(pageBuilder, column, defaultValue, formatter);
 
-            } else if (type instanceof TimestampType) {
+            }
+            else if (type instanceof TimestampType) {
                 // TODO use flexible time format like Ruby's Time.parse
                 TimestampParser parser = timestampParsers[column.getIndex()];
                 setter = new TimestampColumnSetter(pageBuilder, column, defaultValue, parser);
 
-            } else if (type instanceof JsonType) {
+            }
+            else if (type instanceof JsonType) {
                 TimestampFormatter formatter = new TimestampFormatter(formatterTask,
                         Optional.of(c.getOption().loadConfig(TimestampColumnOption.class)));
                 setter = new JsonColumnSetter(pageBuilder, column, defaultValue, formatter);
 
-            } else {
-                throw new ConfigException("Unknown column type: "+type);
+            }
+            else {
+                throw new ConfigException("Unknown column type: " + type);
             }
 
             builder.put(column, setter);
@@ -317,10 +394,12 @@ public class MsgpackParserPlugin
                 BigInteger bi = unpacker.unpackBigInteger();
                 if (0 <= bi.compareTo(LONG_MIN) && bi.compareTo(LONG_MAX) <= 0) {
                     setter.set(bi.longValue());
-                } else {
+                }
+                else {
                     setter.setNull();  // TODO set default value
                 }
-            } else {
+            }
+            else {
                 setter.set(unpacker.unpackLong());
             }
             break;
@@ -372,14 +451,16 @@ public class MsgpackParserPlugin
             int n;
             try {
                 n = unpacker.unpackArrayHeader();
-            } catch (MessageInsufficientBufferException ex) {
+            }
+            catch (MessageInsufficientBufferException ex) {
                 // TODO EOFException?
                 return false;
             }
             for (int i = 0; i < n; i++) {
                 if (i < columnSetters.length) {
                     unpackToSetter(unpacker, columnSetters[i]);
-                } else {
+                }
+                else {
                     unpacker.skipValue();
                 }
             }
@@ -405,7 +486,8 @@ public class MsgpackParserPlugin
             int n;
             try {
                 n = unpacker.unpackMapHeader();
-            } catch (MessageInsufficientBufferException ex) {
+            }
+            catch (MessageInsufficientBufferException ex) {
                 // TODO EOFException?
                 return false;
             }
@@ -421,7 +503,8 @@ public class MsgpackParserPlugin
                 DynamicColumnSetter setter = columnSetters.get(key);
                 if (setter != null) {
                     unpackToSetter(unpacker, setter);
-                } else {
+                }
+                else {
                     unpacker.skipValue();
                 }
             }
@@ -455,7 +538,8 @@ public class MsgpackParserPlugin
                     offset += 1;
                 }
                 return 0;
-            } else {
+            }
+            else {
                 return o1.size() - o2.size();
             }
         }

data/src/test/java/org/embulk/parser/msgpack/TestMsgpackParserPlugin.java

@@ -0,0 +1,465 @@
+package org.embulk.parser.msgpack;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.parser.msgpack.MsgpackParserPlugin;
+import org.embulk.parser.msgpack.MsgpackParserPlugin.FileEncoding;
+import org.embulk.parser.msgpack.MsgpackParserPlugin.PluginTask;
+import org.embulk.parser.msgpack.MsgpackParserPlugin.RowEncoding;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.Pages;
+import org.embulk.standards.LocalFileInputPlugin;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.msgpack.core.MessagePack;
+import org.msgpack.core.MessagePacker;
+import org.msgpack.value.ArrayValue;
+import org.msgpack.value.Value;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestMsgpackParserPlugin
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource config;
+    private Random random;
+    private MsgpackParserPlugin plugin;
+    private FileInputRunner runner;
+    private MockPageOutput output;
+
+    @Before
+    public void createResources()
+    {
+        config = config().set("type", "msgpack");
+        random = runtime.getRandom();
+        plugin = new MsgpackParserPlugin();
+        runner = new FileInputRunner(new LocalFileInputPlugin());
+        output = new MockPageOutput();
+    }
+
+    @Test
+    public void checkDefaultValues()
+    {
+        ConfigSource config = this.config.deepCopy();
+        PluginTask task = config.loadConfig(PluginTask.class);
+        assertEquals(FileEncoding.SEQUENCE, task.getFileEncoding());
+        assertEquals(RowEncoding.MAP, task.getRowEncoding());
+
+        // columns
+        SchemaConfig schemaConfig = plugin.getSchemaConfig(task);
+        assertEquals(1, schemaConfig.getColumnCount());
+        assertEquals(Types.JSON, schemaConfig.getColumnType(0));
+    }
+
+    @Test(expected = ConfigException.class)
+    public void throwConfigErrorByInvalidFileEncoding()
+    {
+        ConfigSource config = this.config.deepCopy()
+                .set("columns", sampleSchema())
+                .set("file_encoding", "invalid");
+        config.loadConfig(PluginTask.class);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void throwConfigErrorByInvalidRowEncoding()
+    {
+        ConfigSource config = this.config.deepCopy()
+                .set("columns", sampleSchema())
+                .set("row_encoding", "invalid");
+        config.loadConfig(PluginTask.class);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void throwConfigErrorIfSchemalessWithInvalidRowEncoding()
+    {
+        ConfigSource config = this.config.deepCopy()
+                .set("row_encoding", "invalid");
+        config.loadConfig(PluginTask.class);
+    }
+
+    @Test
+    public void parseArrayArray()
+            throws IOException
+    {
+        SchemaConfig schema = schema(
+                column("_c_boolean", Types.BOOLEAN),
+                column("_c_string", Types.STRING),
+                column("_c_json", Types.JSON),
+                column("_c_double", Types.DOUBLE),
+                column("_c_long", Types.LONG),
+                column("_c_timestamp", Types.TIMESTAMP, config().set("format", "%Y-%m-%d %H:%M:%S"))
+        );
+        ConfigSource config = this.config.deepCopy()
+                .set("columns", schema)
+                .set("file_encoding", "array")
+                .set("row_encoding", "array");
+
+        boolean vBoolean = random.nextBoolean();
+        String vString = nextString(random, random.nextInt(100));
+        double vDouble = random.nextDouble();
+        long vLong = random.nextLong();
+        String vJson = nextString(random, random.nextInt(100));
+        long vTimestamp = nextUnixtime(random, "2013-01-01 00:00:00", 1000);
+
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            try (MessagePacker pk = MessagePack.newDefaultPacker(out)) {
+                pk.packArrayHeader(1)
+                        .packArrayHeader(schema.getColumnCount()) // 1 record
+                        .packBoolean(vBoolean)
+                        .packString(vString)
+                        .packString(vJson)
+                        .packDouble(vDouble)
+                        .packLong(vLong)
+                        .packLong(vTimestamp);
+            }
+
+            try (FileInput in = input(out.toByteArray())) {
+                transaction(config, input(out.toByteArray()), output);
+            }
+        }
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(1, records.size());
+        for (Object[] record : records) {
+            assertEquals(schema.getColumnCount(), record.length);
+            assertEquals(vBoolean, record[0]);
+            assertEquals(vString, record[1]);
+            assertEquals(vJson, ((Value) record[2]).asStringValue().asString());
+            assertEquals(vDouble, (double) record[3], 0.001);
+            assertEquals(vLong, record[4]);
+            assertEquals(vTimestamp, ((Timestamp) record[5]).getEpochSecond());
+        }
+    }
+
+    @Test
+    public void parseSequenceArray()
+            throws IOException
+    {
+        SchemaConfig schema = schema(
+                column("_c_boolean", Types.BOOLEAN),
+                column("_c_string", Types.STRING),
+                column("_c_json", Types.JSON),
+                column("_c_double", Types.DOUBLE),
+                column("_c_long", Types.LONG),
+                column("_c_timestamp", Types.TIMESTAMP, config().set("format", "%Y-%m-%d %H:%M:%S"))
+        );
+        ConfigSource config = this.config.deepCopy()
+                .set("columns", schema)
+                .set("file_encoding", "sequence")
+                .set("row_encoding", "array");
+
+        boolean vBoolean = random.nextBoolean();
+        String vString = nextString(random, random.nextInt(100));
+        double vDouble = random.nextDouble();
+        long vLong = random.nextLong();
+        String vJson = nextString(random, random.nextInt(100));
+        long vTimestamp = nextUnixtime(random, "2013-01-01 00:00:00", 1000);
+
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            try (MessagePacker pk = MessagePack.newDefaultPacker(out)) {
+                pk.packArrayHeader(schema.getColumnCount()) // 1 record
+                        .packBoolean(vBoolean)
+                        .packString(vString)
+                        .packString(vJson)
+                        .packDouble(vDouble)
+                        .packLong(vLong)
+                        .packLong(vTimestamp);
+            }
+
+            try (FileInput in = input(out.toByteArray())) {
+                transaction(config, input(out.toByteArray()), output);
+            }
+        }
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(1, records.size());
+        for (Object[] record : records) {
+            assertEquals(schema.getColumnCount(), record.length);
+            assertEquals(vBoolean, record[0]);
+            assertEquals(vString, record[1]);
+            assertEquals(vJson, ((Value) record[2]).asStringValue().asString());
+            assertEquals(vDouble, (double) record[3], 0.001);
+            assertEquals(vLong, record[4]);
+            assertEquals(vTimestamp, ((Timestamp) record[5]).getEpochSecond());
+        }
+    }
+
+    @Test
+    public void parseSequentialSchemalessData()
+            throws IOException
+    {
+        SchemaConfig schema = schema(column("record", Types.JSON));
+        ConfigSource config = this.config.deepCopy().set("file_encoding", "sequence");
+
+        boolean vBoolean = random.nextBoolean();
+        String vString = nextString(random, random.nextInt(100));
+        double vDouble = random.nextDouble();
+        long vLong = random.nextLong();
+        String vJson = nextString(random, random.nextInt(100));
+
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            try (MessagePacker pk = MessagePack.newDefaultPacker(out)) {
+                pk.packArrayHeader(5) // 1 record
+                        .packBoolean(vBoolean)
+                        .packString(vString)
+                        .packString(vJson)
+                        .packDouble(vDouble)
+                        .packLong(vLong);
+            }
+
+            try (FileInput in = input(out.toByteArray())) {
+                transaction(config, input(out.toByteArray()), output);
+            }
+        }
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(1, records.size());
+        for (Object[] record : records) {
+            assertEquals(1, record.length);
+            assertTrue(((Value) record[0]).isArrayValue());
+            ArrayValue v = ((Value) record[0]).asArrayValue();
+            assertEquals(vBoolean, v.get(0).asBooleanValue().getBoolean());
+            assertEquals(vString, v.get(1).asStringValue().asString());
+            assertEquals(vJson, v.get(2).asStringValue().asString());
+            assertEquals(vDouble, v.get(3).asFloatValue().toDouble(), 0.001);
+            assertEquals(vLong, v.get(4).asIntegerValue().toLong());
+        }
+    }
+
+    @Test
+    public void parseSequenceMap()
+            throws IOException
+    {
+        SchemaConfig schema = schema(
+                column("_c_boolean", Types.BOOLEAN),
+                column("_c_string", Types.STRING),
+                column("_c_json", Types.JSON),
+                column("_c_double", Types.DOUBLE),
+                column("_c_long", Types.LONG),
+                column("_c_timestamp", Types.TIMESTAMP, config().set("format", "%Y-%m-%d %H:%M:%S"))
+        );
+        ConfigSource config = this.config.deepCopy()
+                .set("columns", schema)
+                .set("file_encoding", "sequence")
+                .set("row_encoding", "map");
+
+        boolean vBoolean = random.nextBoolean();
+        String vString = nextString(random, random.nextInt(100));
+        double vDouble = random.nextDouble();
+        long vLong = random.nextLong();
+        String vJson = nextString(random, random.nextInt(100));
+        long vTimestamp = nextUnixtime(random, "2013-01-01 00:00:00", 1000);
+
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            try (MessagePacker pk = MessagePack.newDefaultPacker(out)) {
+                pk.packMapHeader(schema.getColumnCount()) // 1 record
+                        .packString(schema.getColumnName(0)).packBoolean(vBoolean)
+                        .packString(schema.getColumnName(1)).packString(vString)
+                        .packString(schema.getColumnName(2)).packString(vJson)
+                        .packString(schema.getColumnName(3)).packDouble(vDouble)
+                        .packString(schema.getColumnName(4)).packLong(vLong)
+                        .packString(schema.getColumnName(5)).packLong(vTimestamp);
+            }
+
+            try (FileInput in = input(out.toByteArray())) {
+                transaction(config, input(out.toByteArray()), output);
+            }
+        }
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(1, records.size());
+        for (Object[] record : records) {
+            assertEquals(schema.getColumnCount(), record.length);
+            assertEquals(vBoolean, record[0]);
+            assertEquals(vString, record[1]);
+            assertEquals(vJson, ((Value) record[2]).asStringValue().asString());
+            assertEquals(vDouble, (double) record[3], 0.001);
+            assertEquals(vLong, record[4]);
+            assertEquals(vTimestamp, ((Timestamp) record[5]).getEpochSecond());
+        }
+    }
+
+    @Test
+    public void parseArrayMap()
+            throws IOException
+    {
+        SchemaConfig schema = schema(
+                column("_c_boolean", Types.BOOLEAN),
+                column("_c_string", Types.STRING),
+                column("_c_json", Types.JSON),
+                column("_c_double", Types.DOUBLE),
+                column("_c_long", Types.LONG),
+                column("_c_timestamp", Types.TIMESTAMP, config().set("format", "%Y-%m-%d %H:%M:%S"))
+        );
+        ConfigSource config = this.config.deepCopy()
+                .set("columns", schema)
+                .set("file_encoding", "array")
+                .set("row_encoding", "map");
+
+        boolean vBoolean = random.nextBoolean();
+        String vString = nextString(random, random.nextInt(100));
+        double vDouble = random.nextDouble();
+        long vLong = random.nextLong();
+        String vJson = nextString(random, random.nextInt(100));
+        long vTimestamp = nextUnixtime(random, "2013-01-01 00:00:00", 1000);
+
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            try (MessagePacker pk = MessagePack.newDefaultPacker(out)) {
+                pk.packArrayHeader(1)
+                        .packMapHeader(schema.getColumnCount()) // 1 record
+                        .packString(schema.getColumnName(0)).packBoolean(vBoolean)
+                        .packString(schema.getColumnName(1)).packString(vString)
+                        .packString(schema.getColumnName(2)).packString(vJson)
+                        .packString(schema.getColumnName(3)).packDouble(vDouble)
+                        .packString(schema.getColumnName(4)).packLong(vLong)
+                        .packString(schema.getColumnName(5)).packLong(vTimestamp);
+            }
+
+            try (FileInput in = input(out.toByteArray())) {
+                transaction(config, input(out.toByteArray()), output);
+            }
+        }
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(1, records.size());
+        for (Object[] record : records) {
+            assertEquals(schema.getColumnCount(), record.length);
+            assertEquals(vBoolean, record[0]);
+            assertEquals(vString, record[1]);
+            assertEquals(vJson, ((Value) record[2]).asStringValue().asString());
+            assertEquals(vDouble, (double) record[3], 0.001);
+            assertEquals(vLong, record[4]);
+            assertEquals(vTimestamp, ((Timestamp) record[5]).getEpochSecond());
+        }
+    }
+
+    @Test
+    public void parseArraySchemalessData()
+            throws IOException
+    {
+        SchemaConfig schema = schema(column("record", Types.JSON));
+        ConfigSource config = this.config.deepCopy().set("file_encoding", "array");
+
+        boolean vBoolean = random.nextBoolean();
+        String vString = nextString(random, random.nextInt(100));
+        double vDouble = random.nextDouble();
+        long vLong = random.nextLong();
+        String vJson = nextString(random, random.nextInt(100));
+
+        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            try (MessagePacker pk = MessagePack.newDefaultPacker(out)) {
+                pk.packArrayHeader(1)
+                        .packArrayHeader(5) // 1 record
+                        .packBoolean(vBoolean)
+                        .packString(vString)
+                        .packString(vJson)
+                        .packDouble(vDouble)
+                        .packLong(vLong);
+            }
+
+            try (FileInput in = input(out.toByteArray())) {
+                transaction(config, input(out.toByteArray()), output);
+            }
+        }
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(1, records.size());
+        for (Object[] record : records) {
+            assertEquals(1, record.length);
+            assertTrue(((Value) record[0]).isArrayValue());
+            ArrayValue v = ((Value) record[0]).asArrayValue();
+            assertEquals(vBoolean, v.get(0).asBooleanValue().getBoolean());
+            assertEquals(vString, v.get(1).asStringValue().asString());
+            assertEquals(vJson, v.get(2).asStringValue().asString());
+            assertEquals(vDouble, v.get(3).asFloatValue().toDouble(), 0.001);
+            assertEquals(vLong, v.get(4).asIntegerValue().toLong());
+        }
+    }
+
+    private ConfigSource config()
+    {
+        return runtime.getExec().newConfigSource();
+    }
+
+    private SchemaConfig sampleSchema()
+    {
+        return schema(column("_c0", Types.STRING));
+    }
+
+    private SchemaConfig schema(ColumnConfig... columns)
+    {
+        return new SchemaConfig(Lists.newArrayList(columns));
+    }
+
+    private ColumnConfig column(String name, Type type)
+    {
+        return column(name, type, config());
+    }
+
+    private ColumnConfig column(String name, Type type, ConfigSource config)
+    {
+        return new ColumnConfig(name, type, config);
+    }
+
+    private void transaction(ConfigSource config, final FileInput input, final MockPageOutput output)
+    {
+        plugin.transaction(config, new ParserPlugin.Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema schema)
+            {
+                plugin.run(taskSource, schema, input, output);
+            }
+        });
+    }
+
+    private FileInput input(byte[] bytes)
+    {
+        return new InputStreamFileInput(runtime.getBufferAllocator(), provider(new ByteArrayInputStream(bytes)));
+    }
+
+    private InputStreamFileInput.IteratorProvider provider(InputStream... inputStreams)
+    {
+        return new InputStreamFileInput.IteratorProvider(ImmutableList.copyOf(inputStreams));
+    }
+
+    private static String nextString(Random random, int lengthBound)
+    {
+        char[] text = new char[lengthBound];
+        for (int i = 0; i < text.length; i++) {
+            text[i] = (char) random.nextInt(255);
+        }
+        return new String(text);
+    }
+
+    private static long nextUnixtime(Random random, String baseTime, int bound)
+    {
+        long baseUnixtime = java.sql.Timestamp.valueOf(baseTime).getTime();
+        return baseUnixtime + random.nextInt(bound);
+    }
+}

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-msgpack
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-24 00:00:00.000000000 Z
+date: 2016-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Parses files encoded in MessagePack.
 email:
 - frsyuki@gmail.com
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - COPYING
 - ChangeLog
 - README.md
@@ -59,8 +60,8 @@ files:
 - lib/embulk/guess/msgpack.rb
 - lib/embulk/parser/msgpack.rb
 - src/main/java/org/embulk/parser/msgpack/MsgpackParserPlugin.java
-- src/test/java/org/embulk/parser/TestMsgpackParserPlugin.java
-- classpath/embulk-parser-msgpack-0.2.1.jar
+- src/test/java/org/embulk/parser/msgpack/TestMsgpackParserPlugin.java
+- classpath/embulk-parser-msgpack-0.2.2.jar
 homepage: https://github.com/frsyuki/embulk-parser-msgpack
 licenses:
 - Apache 2.0

data/src/test/java/org/embulk/parser/TestMsgpackParserPlugin.java

@@ -1,5 +0,0 @@
-package org.embulk.parser;
-
-public class TestMsgpackParserPlugin
-{
-}