embulk-parser-jsonl 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f98fb168c55e6a4c80f5d098bc72945091fdcf8d
-  data.tar.gz: 5e85f4e3d5aece158e1b55e381d40ee6eda1d69d
+  metadata.gz: 973fe894b7704f01da0d8d0ba2b93eaa3804ea7d
+  data.tar.gz: fe6ca73d3100595bd64e95e0e08269490671314e
 SHA512:
-  metadata.gz: 6452226f7de14018279af312af6f4105dfd98668506168f3acb4e3139519b6a93353e1bac454c0bc272bca96375c4a6fedb201b59f9b639ca0ec7b7774bd69bb
-  data.tar.gz: 1fa70b407a8ad5d8fa37a6467e621aa7e8fd937cedb908d85f3189bcb2afd6506b476c62433bc4ef47ad241b31fc47b46dfc50c77388d504294348540014c09d
+  metadata.gz: 74ad34c4f29980e75f36ac88b9160fd784a975a83b906d6d5c30738edab617211c0653f22d0a9c01ad533bbcc92394b3ceccad20dcc8805b1627298695b5d6ac
+  data.tar.gz: dd611c8bff080d4ca1c61a8c071a5baeb2d982b89b2812457af67f9ecaacd5c3e17b42c230356087c26ed53ecb51a7a0728a1fde14c67a441974483a53fcd58b

data/.gitignore

@@ -2,4 +2,7 @@
 /pkg/
 /tmp/
 /.bundle/
+build/
+/classpath/
+/.gradle
 /Gemfile.lock

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+## 0.1.0 - 2016-02-22
+
+Upgrade Embulk v0.8 and support Json type in Java [#3](https://github.com/shun0102/embulk-parser-jsonl/pull/3)
+
+## 0.0.1 - 2015-04-04
+
+The first release!!

data/README.md

@@ -10,7 +10,7 @@ TODO: Write short description here and embulk-parser-jsonl.gemspec file.
 ## Configuration
 
 - **type**: specify this parser as jsonl
-- **schema**: specify column name and type (array, required)
+- **columns**: specify column name and type (array, required)
 
 ## Example
 
@@ -19,7 +19,7 @@ in:
   type: any file input plugin type
   parser:
     type: jsonl
-    schema:
+    columns:
       - {name: first_name, type: string}
       - {name: last_name, type: string}
       - {name: age, type: long}
@@ -35,5 +35,5 @@ $ embulk guess -g jsonl config.yml -o guessed.yml
 ## Build
 
 ```
-$ rake
+$ ./gradlew gem classpath
 ```

data/build.gradle

@@ -0,0 +1,79 @@
+plugins {
+    id "com.jfrog.bintray" version "1.1"
+    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "java"
+    id "jacoco"
+}
+import com.github.jrubygradle.JRubyExec
+repositories {
+    mavenCentral()
+    jcenter()
+}
+configurations {
+    provided
+}
+
+version = "0.1.0"
+
+compileJava.options.encoding = 'UTF-8' // source encoding
+sourceCompatibility = 1.7
+targetCompatibility = 1.7
+
+dependencies {
+    compile  "org.embulk:embulk-core:0.8.2"
+    provided "org.embulk:embulk-core:0.8.2"
+
+    testCompile "junit:junit:4.+"
+    testCompile  "org.embulk:embulk-core:0.8.2:tests"
+    testCompile  "org.embulk:embulk-standards:0.8.2"
+}
+
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file("classpath").deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into "classpath"
+}
+clean { delete "classpath" }
+
+task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
+    script "${project.name}.gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+
+task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
+    script "pkg/${project.name}-${project.version}.gem"
+}
+
+task "package"(dependsOn: ["gemspec", "classpath"]) << {
+    println "> Build succeeded."
+    println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+}
+
+task gemspec {
+    ext.gemspecFile = file("${project.name}.gemspec")
+    inputs.file "build.gradle"
+    outputs.file gemspecFile
+    doLast { gemspecFile.write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = ["Shunsuke Mikami"]
+  spec.summary       = "Jsonl parser plugin for Embulk"
+  spec.description   = "Parses Jsonl files read by other file input plugins."
+  spec.email         = ["shun0102@gmail.com"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/shun0102/embulk-parser-jsonl"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['~> 10.0']
+end
+/$)
+    }
+}
+clean { delete "${project.name}.gemspec" }

data/embulk-parser-jsonl.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-parser-jsonl"
-  spec.version       = "0.0.1"
+  spec.version       = "0.1.0"
   spec.authors       = ["Shunsuke Mikami"]
   spec.summary       = "Jsonl parser plugin for Embulk"
   spec.description   = "Parses Jsonl files read by other file input plugins."

data/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Tue Aug 11 00:26:20 PDT 2015
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip

data/gradlew

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn ( ) {
+    echo "$*"
+}
+
+die ( ) {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+esac
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched.
+if $cygwin ; then
+    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+fi
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >&-
+APP_HOME="`pwd -P`"
+cd "$SAVED" >&-
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
+function splitJvmOpts() {
+    JVM_OPTS=("$@")
+}
+eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
+JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+
+exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"

data/gradlew.bat

@@ -0,0 +1,90 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/guess/jsonl.rb

@@ -1,84 +1,33 @@
 require 'json'
+require "embulk/parser/jsonl.rb"
 
 module Embulk
   module Guess
+    # $ embulk guess -g "jsonl" partial-config.yml
 
-    # TODO implement guess plugin to make this command work:
-    #      $ embulk guess -g "jsonl" partial-config.yml
-    #
-    #      Depending on the file format the plugin uses, you can use choose
-    #      one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
-    #      or line guess (LineGuessPlugin).
-
-    require "embulk/parser/jsonl.rb"
-
-    #class JsonlParserGuessPlugin < GuessPlugin
-    #  Plugin.register_guess("jsonl", self)
-    #
-    #  def guess(config, sample_buffer)
-    #    if sample_buffer[0,2] == GZIP_HEADER
-    #      guessed = {}
-    #      guessed["type"] = "jsonl"
-    #      guessed["property1"] = "guessed-value"
-    #      return {"parser" => guessed}
-    #    else
-    #      return {}
-    #    end
-    #  end
-    #end
-
-    #class JsonlParserGuessPlugin < TextGuessPlugin
-    #  Plugin.register_guess("jsonl", self)
-    #
-    #  def guess_text(config, sample_text)
-    #    js = JSON.parse(sample_text) rescue nil
-    #    if js && js["mykeyword"] == "keyword"
-    #      guessed = {}
-    #      guessed["type"] = "jsonl"
-    #      guessed["property1"] = "guessed-value"
-    #      return {"parser" => guessed}
-    #    else
-    #      return {}
-    #    end
-    #  end
-    #end
-
-    class JsonlParserGuessPlugin < LineGuessPlugin
+    class Jsonl < LineGuessPlugin # TODO should use GuessPlugin instead of LineGuessPlugin
       Plugin.register_guess("jsonl", self)
 
       def guess_lines(config, sample_lines)
+        #return {} unless config.fetch("parser", {}).fetch("type", "jsonl") == "jsonl"
+
+        rows = []
+
         columns = {}
         sample_lines.each do |line|
-          hash = JSON.parse(line)
-          hash.each do |k, v|
-            columns[k] = get_embulk_type(v)
-          end
+          rows << JSON.parse(line)
         end
-        schema = []
-        columns.each do |k,v|
-          schema << {'name' => k, 'type' => v}
-        end
-        guessed = {}
-        guessed["type"] = "jsonl"
-        guessed["schema"] = schema
-        return {"parser" => guessed}
-      end
 
-      private
+        return {} if rows.size <= 3
 
-      def get_embulk_type(val)
-        case val
-        when TrueClass
-          return "boolean"
-        when FalseClass
-          return "boolean"
-        when Integer
-          return "long"
-        when Float
-          return "double"
-        else
-          return "string"
+        columns = Embulk::Guess::SchemaGuess.from_hash_records(rows).map do |c|
+          column = {name: c.name, type: c.type}
+          column[:format] = c.format if c.format
+          column
         end
+        parser_guessed = {"type" => "jsonl"}
+        parser_guessed["columns"] = columns
+        return {"parser" => parser_guessed}
       end
     end
   end

data/lib/embulk/parser/jsonl.rb

@@ -1,67 +1,3 @@
-require 'json'
-
-module Embulk
-  module Parser
-
-    class JsonlParserPlugin < ParserPlugin
-      Plugin.register_parser("jsonl", self)
-
-      def self.transaction(config, &control)
-        parser_task = config.load_config(Java::LineDecoder::DecoderTask)
-        task = {
-          "decoder_task" => DataSource.from_java(parser_task.dump),
-          "schema" => config.param("schema", :array)
-        }
-        columns = task["schema"].each_with_index.map do |c, i|
-          Column.new(i, c["name"], c["type"].to_sym)
-        end
-        yield(task, columns)
-      end
-
-      def init
-        @decoder_task = task.param("decoder_task", :hash).load_task(Java::LineDecoder::DecoderTask)
-      end
-
-      def run(file_input)
-        decoder = Java::LineDecoder.new(file_input.instance_eval { @java_file_input }, @decoder_task)
-        schema = @task["schema"]
-
-        while decoder.nextFile
-          while line = decoder.poll
-            begin
-              hash = JSON.parse(line)
-              @page_builder.add(make_record(schema, hash))
-            rescue
-              # TODO: logging
-            end
-          end
-        end
-        page_builder.finish
-      end
-
-      private
-
-      def make_record(schema, e)
-        schema.map do |c|
-          val = e[c["name"]]
-          v = val.nil? ? "" : val
-          case c["type"]
-          when "string"
-            v
-          when "long"
-            v.to_i
-          when "double"
-            v.to_f
-          when "boolean"
-            ["yes", "true", "1"].include?(v.downcase)
-          when "timestamp"
-            v.empty? ? nil : Time.strptime(v, c["time_format"])
-          else
-            raise "Unsupported type #{c['type']}"
-          end
-        end
-      end
-    end
-
-  end
-end
+Embulk::JavaPlugin.register_parser(
+  "jsonl", "org.embulk.parser.jsonl.JsonlParserPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = 'embulk-parser-jsonl'

data/src/main/java/org/embulk/parser/jsonl/JsonlParserPlugin.java

@@ -0,0 +1,282 @@
+package org.embulk.parser.jsonl;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableMap;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.util.LineDecoder;
+import org.embulk.spi.util.Timestamps;
+import org.msgpack.core.MessageTypeException;
+import org.msgpack.value.BooleanValue;
+import org.msgpack.value.FloatValue;
+import org.msgpack.value.IntegerValue;
+import org.msgpack.value.Value;
+import org.slf4j.Logger;
+
+import java.util.Map;
+
+import static org.msgpack.value.ValueFactory.newString;
+
+public class JsonlParserPlugin
+        implements ParserPlugin
+{
+    public interface PluginTask
+            extends Task, LineDecoder.DecoderTask, TimestampParser.Task
+    {
+        @Config("columns")
+        @ConfigDefault("null")
+        Optional<SchemaConfig> getSchemaConfig();
+
+        @Config("schema")
+        @ConfigDefault("null")
+        @Deprecated
+        Optional<SchemaConfig> getOldSchemaConfig();
+
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
+    }
+
+    private final Logger log;
+
+    private String line = null;
+    private long lineNumber = 0;
+    private Map<String, Value> columnNameValues;
+
+    public JsonlParserPlugin()
+    {
+        this.log = Exec.getLogger(JsonlParserPlugin.class);
+    }
+
+    @Override
+    public void transaction(ConfigSource configSource, Control control)
+    {
+        PluginTask task = configSource.loadConfig(PluginTask.class);
+        control.run(task.dump(), getSchemaConfig(task).toSchema());
+    }
+
+    // this method is to keep the backward compatibility of 'schema' option.
+    private SchemaConfig getSchemaConfig(PluginTask task)
+    {
+        if (task.getOldSchemaConfig().isPresent()) {
+            log.warn("Please use 'columns' option instead of 'schema' because the 'schema' option is deprecated. The next version will stop 'schema' option support.");
+        }
+
+        if (task.getSchemaConfig().isPresent()) {
+            return task.getSchemaConfig().get();
+        }
+        else if (task.getOldSchemaConfig().isPresent()) {
+            return task.getOldSchemaConfig().get();
+        }
+        else {
+            throw new ConfigException("Attribute 'columns' is required but not set");
+        }
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        setColumnNameValues(schema);
+
+        final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, getSchemaConfig(task));
+        final LineDecoder decoder = newLineDecoder(input, task);
+        final JsonParser jsonParser = newJsonParser();
+        final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
+
+        try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
+            while (decoder.nextFile()) { // TODO this implementation should be improved with new JsonParser API on Embulk v0.8.3
+                lineNumber = 0;
+
+                while ((line = decoder.poll()) != null) {
+                    lineNumber++;
+
+                    try {
+                        Value value = jsonParser.parse(line);
+
+                        if (!value.isMapValue()) {
+                            throw new JsonRecordValidateException("Json string is not representing map value.");
+                        }
+
+                        final Map<Value, Value> record = value.asMapValue().map();
+
+                        schema.visitColumns(new ColumnVisitor() {
+                            @Override
+                            public void booleanColumn(Column column)
+                            {
+                                Value v = record.get(getColumnNameValue(column));
+                                if (isNil(v)) {
+                                    pageBuilder.setNull(column);
+                                }
+                                else {
+                                    try {
+                                        pageBuilder.setBoolean(column, ((BooleanValue) v).getBoolean());
+                                    }
+                                    catch (MessageTypeException e) {
+                                        throw new JsonRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            @Override
+                            public void longColumn(Column column)
+                            {
+                                Value v = record.get(getColumnNameValue(column));
+                                if (isNil(v)) {
+                                    pageBuilder.setNull(column);
+                                }
+                                else {
+                                    try {
+                                        pageBuilder.setLong(column, ((IntegerValue) v).asLong());
+                                    }
+                                    catch (MessageTypeException e) {
+                                        throw new JsonRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            @Override
+                            public void doubleColumn(Column column)
+                            {
+                                Value v = record.get(getColumnNameValue(column));
+                                if (isNil(v)) {
+                                    pageBuilder.setNull(column);
+                                }
+                                else {
+                                    try {
+                                        pageBuilder.setDouble(column, ((FloatValue) v).toDouble());
+                                    }
+                                    catch (MessageTypeException e) {
+                                        throw new JsonRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            @Override
+                            public void stringColumn(Column column)
+                            {
+                                Value v = record.get(getColumnNameValue(column));
+                                if (isNil(v)) {
+                                    pageBuilder.setNull(column);
+                                }
+                                else {
+                                    try {
+                                        pageBuilder.setString(column, v.toString());
+                                    }
+                                    catch (MessageTypeException e) {
+                                        throw new JsonRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            @Override
+                            public void timestampColumn(Column column)
+                            {
+                                Value v = record.get(getColumnNameValue(column));
+                                if (isNil(v)) {
+                                    pageBuilder.setNull(column);
+                                }
+                                else {
+                                    try {
+                                        pageBuilder.setTimestamp(column, timestampParsers[column.getIndex()].parse(v.toString()));
+                                    }
+                                    catch (MessageTypeException e) {
+                                        throw new JsonRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            @Override
+                            public void jsonColumn(Column column)
+                            {
+                                Value v = record.get(getColumnNameValue(column));
+                                if (isNil(v)) {
+                                    pageBuilder.setNull(column);
+                                }
+                                else {
+                                    try {
+                                        pageBuilder.setJson(column, v);
+                                    }
+                                    catch (MessageTypeException e) {
+                                        throw new JsonRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            private boolean isNil(Value v)
+                            {
+                                return v == null || v.isNilValue();
+                            }
+                        });
+
+                        pageBuilder.addRecord();
+                    }
+                    catch (JsonRecordValidateException e) {
+                        if (stopOnInvalidRecord) {
+                            throw new DataException(String.format("Invalid record at line %d: %s", lineNumber, line), e);
+                        }
+                        log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), line));
+                    }
+                }
+            }
+
+            pageBuilder.finish();
+        }
+    }
+
+    private void setColumnNameValues(Schema schema)
+    {
+        ImmutableMap.Builder<String, Value> builder = ImmutableMap.builder();
+        for (Column column : schema.getColumns()) {
+            String name = column.getName();
+            builder.put(name, newString(name));
+        }
+        columnNameValues = builder.build();
+    }
+
+    private Value getColumnNameValue(Column column)
+    {
+        return columnNameValues.get(column.getName());
+    }
+
+    public LineDecoder newLineDecoder(FileInput input, PluginTask task)
+    {
+        return new LineDecoder(input, task);
+    }
+
+    public JsonParser newJsonParser()
+    {
+        return new JsonParser();
+    }
+
+    static class JsonRecordValidateException
+            extends DataException
+    {
+        JsonRecordValidateException(String message)
+        {
+            super(message);
+        }
+
+        JsonRecordValidateException(Throwable cause)
+        {
+            super(cause);
+        }
+    }
+}

data/src/test/java/org/embulk/parser/jsonl/TestJsonlParserPlugin.java

@@ -0,0 +1,228 @@
+package org.embulk.parser.jsonl;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.DataException;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.Pages;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import static org.embulk.spi.type.Types.BOOLEAN;
+import static org.embulk.spi.type.Types.DOUBLE;
+import static org.embulk.spi.type.Types.JSON;
+import static org.embulk.spi.type.Types.LONG;
+import static org.embulk.spi.type.Types.STRING;
+import static org.embulk.spi.type.Types.TIMESTAMP;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.msgpack.value.ValueFactory.newArray;
+import static org.msgpack.value.ValueFactory.newMap;
+import static org.msgpack.value.ValueFactory.newString;
+
+public class TestJsonlParserPlugin
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource config;
+    private JsonlParserPlugin plugin;
+    private MockPageOutput output;
+
+    @Before
+    public void createResource()
+    {
+        config = config().set("type", "jsonl");
+        plugin = new JsonlParserPlugin();
+        recreatePageOutput();
+    }
+
+    private void recreatePageOutput()
+    {
+        output = new MockPageOutput();
+    }
+
+    @Test
+    public void skipRecords()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
+                column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
+        ConfigSource config = this.config.deepCopy().set("columns", schema);
+
+        transaction(config, fileInput(
+                "[]",
+                "\"embulk\"",
+                "10",
+                "true",
+                "false",
+                "null"
+        ));
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(0, records.size());
+    }
+
+    @Test
+    public void throwDataException()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
+                column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
+        ConfigSource config = this.config.deepCopy().set("columns", schema).set("stop_on_invalid_record", true);
+
+        try {
+            transaction(config, fileInput(
+                    "\"not_map_value\""
+            ));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void writeNils()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
+                column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
+        ConfigSource config = this.config.deepCopy().set("columns", schema);
+
+        transaction(config, fileInput(
+                "{}",
+                "{\"_c0\":null,\"_c1\":null,\"_c2\":null}",
+                "{\"_c3\":null,\"_c4\":null,\"_c5\":null}",
+                "{}"
+        ));
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(4, records.size());
+
+        for (Object[] record : records) {
+            for (int i = 0; i < 6; i++) {
+                assertNull(record[i]);
+            }
+        }
+    }
+
+    @Test
+    public void useNormal()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
+                column("_c3", STRING), column("_c4", TIMESTAMP, config().set("format", "%Y-%m-%d %H:%M:%S %Z")), column("_c5", JSON));
+        List<ConfigSource> configs = Lists.newArrayList(
+                this.config.deepCopy().set("columns", schema),
+                this.config.deepCopy().set("schema", schema)
+        );
+
+        for (ConfigSource config : configs) {
+            transaction(config, fileInput(
+                    "{\"_c0\":true,\"_c1\":10,\"_c2\":0.1,\"_c3\":\"embulk\",\"_c4\":\"2016-01-01 00:00:00 UTC\",\"_c5\":{\"k\":\"v\"}}",
+                    "[1, 2, 3]",
+                    "{\"_c0\":false,\"_c1\":-10,\"_c2\":1.0,\"_c3\":\"エンバルク\",\"_c4\":\"2016-01-01 00:00:00 +0000\",\"_c5\":[\"e0\",\"e1\"]}"
+            ));
+
+            List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+            assertEquals(2, records.size());
+
+            Object[] record;
+            {
+                record = records.get(0);
+                assertEquals(true, record[0]);
+                assertEquals(10L, record[1]);
+                assertEquals(0.1, (Double) record[2], 0.0001);
+                assertEquals("embulk", record[3]);
+                assertEquals(Timestamp.ofEpochSecond(1451606400L), record[4]);
+                assertEquals(newMap(newString("k"), newString("v")), record[5]);
+            }
+            {
+                record = records.get(1);
+                assertEquals(false, record[0]);
+                assertEquals(-10L, record[1]);
+                assertEquals(1.0, (Double) record[2], 0.0001);
+                assertEquals("エンバルク", record[3]);
+                assertEquals(Timestamp.ofEpochSecond(1451606400L), record[4]);
+                assertEquals(newArray(newString("e0"), newString("e1")), record[5]);
+            }
+
+            recreatePageOutput();
+        }
+    }
+
+    private ConfigSource config()
+    {
+        return runtime.getExec().newConfigSource();
+    }
+
+    private void transaction(ConfigSource config, final FileInput input)
+    {
+        plugin.transaction(config, new ParserPlugin.Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema schema)
+            {
+                plugin.run(taskSource, schema, input, output);
+            }
+        });
+    }
+
+    private FileInput fileInput(String... lines)
+            throws Exception
+    {
+        StringBuilder sb = new StringBuilder();
+        for (String line : lines) {
+            sb.append(line).append("\n");
+        }
+
+        ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes());
+        return new InputStreamFileInput(runtime.getBufferAllocator(), provider(in));
+    }
+
+    private InputStreamFileInput.IteratorProvider provider(InputStream... inputStreams)
+            throws IOException
+    {
+        return new InputStreamFileInput.IteratorProvider(
+                ImmutableList.copyOf(inputStreams));
+    }
+
+    private SchemaConfig schema(ColumnConfig... columns)
+    {
+        return new SchemaConfig(Lists.newArrayList(columns));
+    }
+
+    private ColumnConfig column(String name, Type type)
+    {
+        return column(name, type, config());
+    }
+
+    private ColumnConfig column(String name, Type type, ConfigSource option)
+    {
+        return new ColumnConfig(name, type, option);
+    }
+}

metadata

@@ -1,41 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-jsonl
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Shunsuke Mikami
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-04 00:00:00.000000000 Z
+date: 2016-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  type: :development
+  name: bundler
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
-  type: :development
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '10.0'
 description: Parses Jsonl files read by other file input plugins.
@@ -45,36 +45,46 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
+- .gitignore
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- build.gradle
 - embulk-parser-jsonl.gemspec
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
 - lib/embulk/guess/jsonl.rb
 - lib/embulk/parser/jsonl.rb
+- settings.gradle
+- src/main/java/org/embulk/parser/jsonl/JsonlParserPlugin.java
+- src/test/java/org/embulk/parser/jsonl/TestJsonlParserPlugin.java
+- classpath/embulk-parser-jsonl-0.1.0.jar
 homepage: https://github.com/shun0102/embulk-parser-jsonl
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
 specification_version: 4
 summary: Jsonl parser plugin for Embulk
 test_files: []