embulk 0.8.4 → 0.8.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 671d7e093dbc15c2d87cf48cc69d6c46db01906f
-  data.tar.gz: 6fea3fc181559ca1821ca48f462cd2944994b741
+  metadata.gz: 99cb7ac6a07d7e8db18b43a95012b56e20ae35cc
+  data.tar.gz: 664f254e002c1d086e73412e67a9efd32c4b4787
 SHA512:
-  metadata.gz: b4c6d18798c4cdc272348c5febdcc875478a2d21165fe37fd9b5c1f55fdd3c97b57aa8260f840cd7e213c11cb65ec1768c6df9c1413f16ea25435b2a50b2bce6
-  data.tar.gz: cee70f19578b92763838f3f404af6cd562aa566ab31548c0d9197c54e27f74bd11fd7ae49c2ed0ed94dd76e25ae20bb7add76e595f20cea0a76995e77ffe895b
+  metadata.gz: 0076dcca57267d34e1474373a56addccbaec0a43b30dc8a23bd504ccca3725802f0e41d90caba94387e0db560090e076829f48bb34789cf8cebaeb4fe7bd93f4
+  data.tar.gz: c0889fa0e4a95b9adb83e4460f4e4821c2c6948ac6583649286e67af305459bd5855df03fa3784da01a048d9390c7218e739e1247ff5d1b84f353047baddd611

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.4'
+    version = '0.8.5'
 
     ext {
         jrubyVersion = '9.0.4.0'

data/embulk-docs/src/built-in.rst

@@ -277,6 +277,40 @@ Example
         - {name: purchase, type: timestamp, format: '%Y%m%d'}
         - {name: comment, type: string}
 
+
+JSON parser plugin
+------------------
+
+The ``json`` parser plugin parses a JSON file that contains a sequence of JSON objects. Example:
+
+.. code-block:: json
+
+    {"time":1455829282,"ip":"93.184.216.34","name":frsyuki}
+    {"time":1455829282,"ip":"172.36.8.109":sadayuki}
+    {"time":1455829284,"ip":"example.com","name":Treasure Data}
+    {"time":1455829282,"ip":"10.98.43.1","name":MessagePack}
+
+``json`` parser plugin outputs a single record named "record" (type is json).
+
+Options
+~~~~~~~~~~~~~~~~~~
+
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| name                       | type     | description                                                                                                    |              required? |
++============================+==========+================================================================================================================+========================+
+| stop\_on\_invalid\_record  | boolean  | Stop bulk load transaction if a file includes invalid record (such as invalid json)                            | ``false`` by default   |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+
+
+Example
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+
+    in:
+      parser:
+        type: json
+
 Gzip decoder plugin
 ------------------
 

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.5
     release/release-0.8.4
     release/release-0.8.3
     release/release-0.8.2

data/embulk-docs/src/release/release-0.8.5.rst

@@ -0,0 +1,11 @@
+Release 0.8.5
+==================================
+
+General Changes
+------------------
+
+* Added ``json`` parser plugin. It doesn't have options to extract values into columns (as like embulk-parser-jsonl plugin supports). It's intended to be supported by a filter plugin.
+
+Release Date
+------------------
+2016-02-18

data/embulk-standards/src/main/java/org/embulk/standards/JsonParserPlugin.java

@@ -0,0 +1,116 @@
+package org.embulk.standards;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.type.Types;
+import org.embulk.spi.util.FileInputInputStream;
+import org.msgpack.value.Value;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+
+public class JsonParserPlugin
+        implements ParserPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
+    }
+
+    private final Logger log;
+
+    public JsonParserPlugin()
+    {
+        this.log = Exec.getLogger(JsonParserPlugin.class);
+    }
+
+    @Override
+    public void transaction(ConfigSource configSource, Control control)
+    {
+        PluginTask task = configSource.loadConfig(PluginTask.class);
+        control.run(task.dump(), newSchema());
+    }
+
+    @VisibleForTesting
+    Schema newSchema()
+    {
+        return Schema.builder().add("record", Types.JSON).build(); // generate a schema
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
+        final Column column = schema.getColumn(0); // record column
+
+        try (PageBuilder pageBuilder = newPageBuilder(schema, output);
+                FileInputInputStream in = new FileInputInputStream(input)) {
+            while (in.nextFile()) {
+                try (JsonParser.Stream stream = newJsonStream(in)) {
+                    Value value;
+                    while ((value = stream.next()) != null) {
+                        try {
+                            if (!value.isMapValue()) {
+                                throw new JsonRecordValidateException(
+                                        String.format("A Json record must not represent map value but it's %s", value.getValueType().name()));
+                            }
+
+                            pageBuilder.setJson(column, value);
+                            pageBuilder.addRecord();
+                        }
+                        catch (JsonRecordValidateException e) {
+                            if (stopOnInvalidRecord) {
+                                throw new DataException(String.format("Invalid record: %s", value.toJson()), e);
+                            }
+                            log.warn(String.format("Skipped record (%s): %s", e.getMessage(), value.toJson()));
+                        }
+                    }
+                }
+                catch (IOException | JsonParseException e) {
+                    throw new DataException(e);
+                }
+            }
+
+            pageBuilder.finish();
+        }
+    }
+
+    private PageBuilder newPageBuilder(Schema schema, PageOutput output)
+    {
+        return new PageBuilder(Exec.getBufferAllocator(), schema, output);
+    }
+
+    private JsonParser.Stream newJsonStream(FileInputInputStream in)
+            throws IOException
+    {
+        return new JsonParser().open(in);
+    }
+
+    static class JsonRecordValidateException
+            extends DataException
+    {
+        JsonRecordValidateException(String message)
+        {
+            super(message);
+        }
+    }
+}

data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java

@@ -27,6 +27,7 @@ public class StandardPluginModule
 
         // parser plugins
         registerPluginTo(binder, ParserPlugin.class, "csv", CsvParserPlugin.class);
+        registerPluginTo(binder, ParserPlugin.class, "json", JsonParserPlugin.class);
 
         // file decoder plugins
         registerPluginTo(binder, DecoderPlugin.class, "gzip", GzipFileDecoderPlugin.class);
@@ -50,6 +51,7 @@ public class StandardPluginModule
         // default guess plugins
         registerDefaultGuessPluginTo(binder, new PluginType("gzip"));
         registerDefaultGuessPluginTo(binder, new PluginType("bzip2"));
+        registerDefaultGuessPluginTo(binder, new PluginType("json")); // should be registered before CsvGuessPlugin
         registerDefaultGuessPluginTo(binder, new PluginType("csv"));
         // charset and newline guess plugins are loaded and invoked by CsvGuessPlugin
     }

data/embulk-standards/src/test/java/org/embulk/standards/TestJsonParserPlugin.java

@@ -0,0 +1,169 @@
+package org.embulk.standards;
+
+import com.google.common.collect.ImmutableList;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.DataException;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.Pages;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.msgpack.value.Value;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.msgpack.value.ValueFactory.newArray;
+import static org.msgpack.value.ValueFactory.newBoolean;
+import static org.msgpack.value.ValueFactory.newInteger;
+import static org.msgpack.value.ValueFactory.newMap;
+import static org.msgpack.value.ValueFactory.newString;
+
+public class TestJsonParserPlugin
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource config;
+    private JsonParserPlugin plugin;
+    private MockPageOutput output;
+
+    @Before
+    public void createResource()
+    {
+        config = config();
+        plugin = new JsonParserPlugin();
+        output = new MockPageOutput();
+    }
+
+    @Test
+    public void readNormalJson()
+            throws Exception
+    {
+        transaction(config, fileInput(
+                "{\"_c0\":true,\"_c1\":10,\"_c2\":\"embulk\",\"_c3\":{\"k\":\"v\"}}",
+                "{}",
+                "{\n" +
+                        "\"_c0\":false,\n" +
+                        "\"_c1\":-10,\n" +
+                        "\"_c2\":\"エンバルク\",\n" +
+                        "\"_c3\":[\"e0\",\"e1\"]\n" +
+                "}",
+                "[1, 2, 3]", // this line should be skipped.
+                "\"embulk\"", // this line should be skipped.
+                "10", // this line should be skipped.
+                "true", // this line should be skipped.
+                "false", // this line should be skipped.
+                "null" // this line should be skipped.
+        ));
+
+        List<Object[]> records = Pages.toObjects(plugin.newSchema(), output.pages);
+        assertEquals(3, records.size());
+
+        Object[] record;
+        Map<Value, Value> map;
+        { // "{\"_c0\":true,\"_c1\":10,\"_c2\":\"embulk\",\"_c3\":{\"k\":\"v\"}}"
+            record = records.get(0);
+            assertEquals(1, record.length);
+            map = ((Value)record[0]).asMapValue().map();
+
+            assertEquals(newBoolean(true), map.get(newString("_c0")));
+            assertEquals(newInteger(10L), map.get(newString("_c1")));
+            assertEquals(newString("embulk"), map.get(newString("_c2")));
+            assertEquals(newMap(newString("k"), newString("v")), map.get(newString("_c3")));
+        }
+        { // "{}"
+            record = records.get(1);
+            assertEquals(1, record.length);
+            assertTrue(((Value)record[0]).asMapValue().map().isEmpty());
+        }
+        {
+            record = records.get(2);
+            assertEquals(1, record.length);
+            map = ((Value)record[0]).asMapValue().map();
+
+            assertEquals(newBoolean(false), map.get(newString("_c0")));
+            assertEquals(newInteger(-10L), map.get(newString("_c1")));
+            assertEquals(newString("エンバルク"), map.get(newString("_c2")));
+            assertEquals(newArray(newString("e0"), newString("e1")), map.get(newString("_c3")));
+        }
+    }
+
+    @Test
+    public void useStopOnInvalidRecord()
+            throws Exception
+    {
+        ConfigSource config = this.config.deepCopy().set("stop_on_invalid_record", true);
+
+        try {
+            transaction(config, fileInput(
+                    "[1, 2, 3]" // throw DataException
+            ));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void readBrokenJson()
+    {
+        try {
+            transaction(config, fileInput(
+                    "{\"_c0\":true,\"_c1\":10," // throw DataException
+            ));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    private ConfigSource config()
+    {
+        return runtime.getExec().newConfigSource();
+    }
+
+    private void transaction(ConfigSource config, final FileInput input)
+    {
+        plugin.transaction(config, new ParserPlugin.Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema schema)
+            {
+                plugin.run(taskSource, schema, input, output);
+            }
+        });
+    }
+
+    private FileInput fileInput(String... lines)
+            throws Exception
+    {
+        StringBuilder sb = new StringBuilder();
+        for (String line : lines) {
+            sb.append(line).append("\n");
+        }
+
+        ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes());
+        return new InputStreamFileInput(runtime.getBufferAllocator(), provider(in));
+    }
+
+    private InputStreamFileInput.IteratorProvider provider(InputStream... inputStreams)
+            throws IOException
+    {
+        return new InputStreamFileInput.IteratorProvider(
+                ImmutableList.copyOf(inputStreams));
+    }
+}

data/lib/embulk/guess/json.rb

@@ -0,0 +1,40 @@
+module Embulk
+  module Guess
+    class JsonGuessPlugin < GuessPlugin
+      Plugin.register_guess('json', self)
+
+      java_import 'com.google.common.collect.Lists'
+      java_import 'java.io.ByteArrayInputStream'
+      java_import 'org.embulk.spi.Exec'
+      java_import 'org.embulk.spi.json.JsonParser'
+      java_import 'org.embulk.spi.json.JsonParseException'
+      java_import 'org.embulk.spi.util.FileInputInputStream'
+      java_import 'org.embulk.spi.util.InputStreamFileInput'
+
+      def guess(config, sample_buffer)
+        return {} unless config.fetch("parser", {}).fetch("type", "json") == "json"
+
+        # Use org.embulk.spi.json.JsonParser to respond to multi-line Json
+        json_parser = new_json_parser(sample_buffer)
+        begin
+          while json_parser.next
+          end
+        rescue JsonParseException
+          return {}
+        end
+
+        return {"parser" => {"type" => "json"}}
+      end
+
+      private
+
+      def new_json_parser(buffer)
+        input_streams = Lists::newArrayList(ByteArrayInputStream.new(buffer.to_java_bytes))
+        iterator_provider = InputStreamFileInput::IteratorProvider.new(input_streams)
+        input = FileInputInputStream.new(InputStreamFileInput.new(Java::SPI::Exec.getBufferAllocator(), iterator_provider))
+        input.nextFile
+        JsonParser.new.open(input)
+      end
+    end
+  end
+end

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.8.4'
+  VERSION = '0.8.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.8.4
+  version: 0.8.5
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-16 00:00:00.000000000 Z
+date: 2016-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jruby-jars
@@ -108,8 +108,8 @@ files:
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-compress-1.10.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.8.4.jar
-- classpath/embulk-standards-0.8.4.jar
+- classpath/embulk-core-0.8.5.jar
+- classpath/embulk-standards-0.8.5.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-bootstrap-0.1.1.jar
@@ -424,6 +424,7 @@ files:
 - embulk-docs/src/release/release-0.8.2.rst
 - embulk-docs/src/release/release-0.8.3.rst
 - embulk-docs/src/release/release-0.8.4.rst
+- embulk-docs/src/release/release-0.8.5.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/Bzip2FileDecoderPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/Bzip2FileEncoderPlugin.java
@@ -432,6 +433,7 @@ files:
 - embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java
 - embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java
+- embulk-standards/src/main/java/org/embulk/standards/JsonParserPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/NullOutputPlugin.java
@@ -443,6 +445,7 @@ files:
 - embulk-standards/src/test/java/org/embulk/standards/TestCsvFormatterPlugin.java
 - embulk-standards/src/test/java/org/embulk/standards/TestCsvParserPlugin.java
 - embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java
+- embulk-standards/src/test/java/org/embulk/standards/TestJsonParserPlugin.java
 - embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java
 - embulk.gemspec
 - gradle/wrapper/gradle-wrapper.jar
@@ -515,6 +518,7 @@ files:
 - lib/embulk/guess/charset.rb
 - lib/embulk/guess/csv.rb
 - lib/embulk/guess/gzip.rb
+- lib/embulk/guess/json.rb
 - lib/embulk/guess/newline.rb
 - lib/embulk/guess/schema_guess.rb
 - lib/embulk/guess/time_format_guess.rb