embulk-parser-header_based_csv 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: '029ace0c5c4d035516de52ffb5e5f706f9c44c42'
+  data.tar.gz: fdb2f0a62854079e593948f5d4bfb2a1c8bcfdb5
+SHA512:
+  metadata.gz: af14e92ce1071d1341c6ec60d421761a6212c66f0cf2c84af74d0a99956eb04f90db4538dbf7a7c21e10b4af36ea851445622ffac1b408c1fc93e752b1661d42
+  data.tar.gz: b3e4a555f6c0f96932a51ca96486693f87d99ba12038f1709e3d203e5688cdff048daaad1c7e7c381346a555ffa2c579b2248de763e511b2432a53309bc45292

data/.gitignore

@@ -0,0 +1,14 @@
+*~
+/pkg/
+/tmp/
+*.gemspec
+.gradle/
+/classpath/
+build/
+.idea
+/.settings/
+/.metadata/
+.classpath
+.project
+*.iml
+/out/

data/.gitlab-ci.yml

@@ -0,0 +1,14 @@
+image: openjdk:8u181
+
+deploy:
+  stage: deploy
+  before_script:
+    - mkdir ~/.gem
+    - echo '---' >> ~/.gem/credentials
+    - echo ':rubygems_api_key:' $RUBYGEMS_API_KEY >> ~/.gem/credentials
+    - chmod 0600 ~/.gem/credentials
+  script:
+    - ./gradlew gemPush
+  only:
+    - tags
+  when: manual

data/.scalafmt.conf

@@ -0,0 +1,7 @@
+style                        = defaultWithAlign
+danglingParentheses          = true
+indentOperator               = spray
+maxColumn                    = 120
+rewrite.rules                = [RedundantParens, SortImports, PreferCurlyFors]
+binPack.literalArgumentLists = false
+unindentTopLevelOperators    = true

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# Header Based Csv parser plugin for Embulk
+
+## Overview
+
+* **Plugin type**: parser
+* **Guess supported**: no
+
+組み込みプラグインのCSVパーサー拡張です  
+
+this parser is an extension of built-in csv parser plugin
+
+このパーサーはパースするとき、ファイルのヘッダー名とコンフィグファイルのカラム名が完全一致しているか調べます  
+そのためファイルのカラムの並び順の変更に影響されずパースできます
+
+when it parses files, it checks that a header name on your file and a column name on your configuration file are exactly matched
+so the change of column order on your files does not affect the parser
+
+CSVファイルにはヘッダーが必要です  
+`skip_header_lines`を指定しない限り、一行目をヘッダーとみなします
+
+csv file must have one header line
+the first line is supposed to be a header line unless you set `skip_header_lines` option 
+
+## Configuration
+
+このプラグイン独自のオプションはありません(CSVパーサーと同じです)  
+ただし`skip_header_lines`を指定する場合、ヘッダー行は含まないようにしてください  
+
+there is no unique option for this plugin (it is the same as that of csv parser plugin)
+but you should be careful not to include its header line when you set `skip_header_lines`
+
+
+## install
+
+```
+$ embulk gem install embulk-parser-header_based_csv
+```
+
+## Build
+
+```
+$ ./gradlew gem  # -t to watch change of files and rebuild continuously
+```

data/build.gradle

@@ -0,0 +1,93 @@
+plugins {
+    id "com.jfrog.bintray" version "1.1"
+    id "com.github.jruby-gradle.base" version "1.5.0"
+    id "scala"
+    id "com.diffplug.gradle.spotless" version "3.14.0"
+}
+import com.github.jrubygradle.JRubyExec
+repositories {
+    mavenCentral()
+    jcenter()
+}
+configurations {
+    provided
+}
+
+version = "0.1.0"
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+dependencies {
+    compile  "org.embulk:embulk-core:0.9.7"
+    compile  "org.embulk:embulk-standards:0.9.7"
+    compile "org.scala-lang:scala-library:2.12.6"
+    testCompile "org.scalatest:scalatest_2.12:3.0.5"
+    testCompile "org.embulk:embulk-core:0.9.7"
+    provided "org.embulk:embulk-core:0.9.7"
+}
+
+spotless {
+    scala {
+        scalafmt('1.5.1').configFile('.scalafmt.conf')
+    }
+}
+
+afterEvaluate {
+    tasks.getByName('spotlessCheck').dependsOn(tasks.getByName('spotlessApply'))
+}
+
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file("classpath").deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into "classpath"
+}
+clean { delete "classpath" }
+
+task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "build", "${project.name}.gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+
+task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "push", "pkg/${project.name}-${project.version}.gem"
+}
+
+task "package"(dependsOn: ["gemspec", "classpath"]) {
+    doLast {
+        println "> Build succeeded."
+        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+    }
+}
+
+task gemspec {
+    ext.gemspecFile = file("${project.name}.gemspec")
+    inputs.file "build.gradle"
+    outputs.file gemspecFile
+    doLast { gemspecFile.write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = ["k_higuchi"]
+  spec.summary       = %[Header Based Csv parser plugin for Embulk]
+  spec.description   = %[Parses Header Based Csv files read by other file input plugins.]
+  spec.email         = ["k_higuchi@septeni-original.co.jp"]
+  spec.licenses      = ["MIT"]
+  # TODO set this: spec.homepage      = "https://github.com/k_higuchi/embulk-parser-header_based_csv"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end
+/$)
+    }
+}
+clean { delete "${project.name}.gemspec" }

data/csv/test_1.csv

@@ -0,0 +1,3 @@
+b,a,c
+2,1,3
+5,4,6

data/csv/test_2.csv

@@ -0,0 +1,3 @@
+a,b,c
+1,2,3
+4,5,6

data/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Tue Aug 21 16:25:32 JST 2018
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip

data/gradlew

@@ -0,0 +1,172 @@
+#!/usr/bin/env sh
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"

data/gradlew.bat

@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/guess/header_based_csv.rb

@@ -0,0 +1,61 @@
+module Embulk
+  module Guess
+
+    # TODO implement guess plugin to make this command work:
+    #      $ embulk guess -g "header_based_csv" partial-config.yml
+    #
+    #      Depending on the file format the plugin uses, you can use choose
+    #      one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
+    #      or line guess (LineGuessPlugin).
+
+    # class HeaderBasedCsv < GuessPlugin
+    #   Plugin.register_guess("header_based_csv", self)
+    #
+    #   def guess(config, sample_buffer)
+    #     if sample_buffer[0,2] == GZIP_HEADER
+    #       guessed = {}
+    #       guessed["type"] = "header_based_csv"
+    #       guessed["property1"] = "guessed-value"
+    #       return {"parser" => guessed}
+    #     else
+    #       return {}
+    #     end
+    #   end
+    # end
+
+    # class HeaderBasedCsv < TextGuessPlugin
+    #   Plugin.register_guess("header_based_csv", self)
+    #
+    #   def guess_text(config, sample_text)
+    #     js = JSON.parse(sample_text) rescue nil
+    #     if js && js["mykeyword"] == "keyword"
+    #       guessed = {}
+    #       guessed["type"] = "header_based_csv"
+    #       guessed["property1"] = "guessed-value"
+    #       return {"parser" => guessed}
+    #     else
+    #       return {}
+    #     end
+    #   end
+    # end
+
+    # class HeaderBasedCsv < LineGuessPlugin
+    #   Plugin.register_guess("header_based_csv", self)
+    #
+    #   def guess_lines(config, sample_lines)
+    #     all_line_matched = sample_lines.all? do |line|
+    #       line =~ /mypattern/
+    #     end
+    #     if all_line_matched
+    #       guessed = {}
+    #       guessed["type"] = "header_based_csv"
+    #       guessed["property1"] = "guessed-value"
+    #       return {"parser" => guessed}
+    #     else
+    #       return {}
+    #     end
+    #   end
+    # end
+
+  end
+end

data/lib/embulk/parser/header_based_csv.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_parser(
+  "header_based_csv", "org.embulk.parser.header_based_csv.HeaderBasedCsvParserPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/scala/helper/GoogleOptionalHelper.scala

@@ -0,0 +1,11 @@
+package helper
+
+import com.google.common.base.Optional
+
+object GoogleOptionalHelper {
+
+  implicit class RichOptional[T](opt: Optional[T]) {
+    def toOption: Option[T] = if (opt.isPresent) Some(opt.get()) else None
+  }
+
+}

data/src/main/scala/org/embulk/parser/header_based_csv/HeaderBasedCsvParserPlugin.scala

@@ -0,0 +1,136 @@
+package org.embulk.parser.header_based_csv
+
+import org.embulk.config._
+import org.embulk.parser.header_based_csv.exception.{ColumnValueNotFoundException, TooManyColumnValuesException}
+import org.embulk.parser.header_based_csv.output.ColumnValueWriter
+import org.embulk.parser.header_based_csv.tokenizer.TokenizerHelper.RichCsvTokenizer
+import org.embulk.spi.util.LineDecoder
+import org.embulk.spi._
+import org.embulk.standards.{CsvParserPlugin, CsvTokenizer}
+import org.slf4j.Logger
+import helper.GoogleOptionalHelper.RichOptional
+
+import scala.annotation.tailrec
+import scala.collection.JavaConverters._
+import scala.util.{Failure, Success, Try}
+
+class HeaderBasedCsvParserPlugin extends CsvParserPlugin {
+
+  trait PluginTask extends CsvParserPlugin.PluginTask
+
+  private val log: Logger = Exec.getLogger(classOf[CsvParserPlugin])
+
+  override def transaction(config: ConfigSource, control: ParserPlugin.Control): Unit = {
+    val task = config.loadConfig(classOf[PluginTask])
+
+    val skipHeaderLines = task.getSkipHeaderLines
+
+    // CsvParserPlugin had header_line option (not anymore)
+    // super class has backward compatibility code in its "transaction" method and it discards the header line when header-line option is true
+    // so you have to override the method and keep the header line
+    task.getHeaderLine.toOption match {
+      case Some(_) if skipHeaderLines > 0 =>
+        throw new ConfigException("'header_line' option is invalid if 'skip_header_lines' is set.")
+      case Some(hasHeader) =>
+        if (hasHeader)
+          task.setSkipHeaderLines(0) // keep the header line (should set 0 not to skip the first line unlike super class)
+        else
+          throw new ConfigException("this parser needs a header line")
+      case None => task.setSkipHeaderLines(0)
+    }
+
+    control.run(task.dump, task.getSchemaConfig.toSchema)
+  }
+
+  override def run(taskSource: TaskSource, schema: Schema, input: FileInput, output: PageOutput): Unit =
+    try {
+      val task = taskSource.loadTask(classOf[PluginTask])
+
+      val allowExtraColumnValues = task.getAllowExtraColumns
+      val allowOptionalColumns   = task.getAllowOptionalColumns
+      val stopOnInvalidRecord    = task.getStopOnInvalidRecord
+
+      val skipHeaderLines = task.getSkipHeaderLines
+
+      val tokenizer         = new CsvTokenizer(new LineDecoder(input, task), task)
+      val pageBuilder       = new PageBuilder(Exec.getBufferAllocator, schema, output)
+      val columnValueWriter = new ColumnValueWriter(task, pageBuilder)
+
+      while (tokenizer.nextFile()) {
+
+        (1 to skipHeaderLines).foreach(_ => tokenizer.skipHeaderLine())
+
+        tokenizer.forEachRecord { record =>
+          val columns = schema.getColumns.asScala.toList
+          val result = writeColumnValues(columns,
+                                         record,
+                                         tokenizer,
+                                         columnValueWriter,
+                                         allowExtraColumnValues,
+                                         allowOptionalColumns)
+          skipOrStopIfFailed(result, tokenizer, stopOnInvalidRecord)
+          if (result.isSuccess) {
+            pageBuilder.addRecord()
+          }
+        }
+        pageBuilder.flush()
+      }
+      pageBuilder.finish()
+    } catch {
+      case e: Throwable => e.printStackTrace()
+    }
+
+  private def writeSingleColumnValue(record: Map[String, String],
+                                     column: Column,
+                                     columnValueWriter: ColumnValueWriter,
+                                     columnCount: Int,
+                                     allowOptionalColumns: Boolean): Try[Unit] =
+    record.get(column.getName) match {
+      case Some(value) =>
+        columnValueWriter.writeValue(column, value)
+
+      case None if allowOptionalColumns && record.size < columnCount =>
+        columnValueWriter.writeNull(column)
+
+      case None =>
+        Failure(new ColumnValueNotFoundException(column))
+    }
+
+  private def writeColumnValues(columns: Seq[Column],
+                                record: Map[String, String],
+                                tokenizer: CsvTokenizer,
+                                columnValueWriter: ColumnValueWriter,
+                                allowExtraColumnValues: Boolean,
+                                allowOptionalColumns: Boolean): Try[Unit] = {
+
+    @tailrec def loop(cs: Seq[Column]): Try[Unit] =
+      cs match {
+        case Nil =>
+          Success(())
+
+        // allowExtraColumns (which renamed to allowExtraColumnValues here because its more suitable)
+        // "If true, ignore too many columns. Otherwise, skip the row in case of too many columns"
+        case _ :: _ if !allowExtraColumnValues && columns.size < record.size =>
+          Failure(new TooManyColumnValuesException(columns.size, record.size))
+
+        case column :: tail =>
+          writeSingleColumnValue(record, column, columnValueWriter, columns.length, allowOptionalColumns) match {
+            case Success(_)    => loop(tail)
+            case f: Failure[_] => f
+          }
+      }
+    loop(columns)
+  }
+
+  private def skipOrStopIfFailed(result: Try[Unit], tokenizer: CsvTokenizer, stopOnInvalidRecord: Boolean): Unit =
+    result.failed
+      .foreach { e =>
+        val skippedLine = tokenizer.skipCurrentLine()
+        val lineNumber  = tokenizer.getCurrentLineNumber
+        if (stopOnInvalidRecord)
+          throw new DataException(s"Invalid record at line $lineNumber: $skippedLine", e)
+        else
+          log.warn(s"Skipped line $lineNumber (${e.getMessage}): $skippedLine")
+      }
+
+}

data/src/main/scala/org/embulk/parser/header_based_csv/exception/ColumnValueNotFoundException.scala

@@ -0,0 +1,6 @@
+package org.embulk.parser.header_based_csv.exception
+
+import org.embulk.spi.{Column, DataException}
+
+class ColumnValueNotFoundException(column: Column)
+    extends DataException(s"a value for this column is not found. column name: ${column.getName}")

data/src/main/scala/org/embulk/parser/header_based_csv/exception/TooManyColumnValuesException.scala

@@ -0,0 +1,8 @@
+package org.embulk.parser.header_based_csv.exception
+
+import org.embulk.spi.DataException
+
+class TooManyColumnValuesException(columnCount: Int, columnValueCount: Int)
+    extends DataException(
+      s"this record has more values than columns in the config file. columns: $columnCount values: $columnValueCount"
+    )

data/src/main/scala/org/embulk/parser/header_based_csv/output/ColumnValueWriter.scala

@@ -0,0 +1,57 @@
+package org.embulk.parser.header_based_csv.output
+
+import org.embulk.parser.header_based_csv.HeaderBasedCsvParserPlugin
+import org.embulk.spi.`type`._
+import org.embulk.spi.json.JsonParser
+import org.embulk.spi.time.TimestampParser
+import org.embulk.spi.util.Timestamps
+import org.embulk.spi.{Column, PageBuilder}
+import helper.GoogleOptionalHelper.RichOptional
+
+import scala.util.{Failure, Try}
+
+class ColumnValueWriter(task: HeaderBasedCsvParserPlugin#PluginTask, pageBuilder: PageBuilder) {
+
+  private lazy val timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig)
+  private lazy val jsonParser       = new JsonParser()
+
+  def writeValue(column: Column, value: String): Try[Unit] = column.getType match {
+    case _: LongType      => writeLong(column, value)
+    case _: DoubleType    => writeDouble(column, value)
+    case _: StringType    => writeString(column, value)
+    case _: TimestampType => writeTimestamp(column, value)(timestampParsers)
+    case _: JsonType      => writeJson(column, value)
+    case _                => Failure(new UnsupportedOperationException(""))
+  }
+
+  def writeNull(column: Column): Try[Unit] = Try(pageBuilder.setNull(column))
+
+  private[output] def writeString(column: Column, value: String): Try[Unit] = {
+    task.getNullString.toOption match {
+      case Some(strToBeNull) if value.matches(strToBeNull) =>
+        Try(pageBuilder.setNull(column))
+      case _ =>
+        Try(pageBuilder.setString(column, value))
+    }
+  }
+
+  private[output] def writeBoolean(column: Column, value: String): Try[Unit] =
+    Try(pageBuilder.setBoolean(column, value.toBoolean))
+
+  private[output] def writeLong(column: Column, value: String): Try[Unit] =
+    Try(pageBuilder.setLong(column, value.toLong))
+
+  private[output] def writeDouble(column: Column, value: String): Try[Unit] =
+    Try(pageBuilder.setDouble(column, value.toDouble))
+
+  private[output] def writeTimestamp(column: Column,
+                                     value: String)(timestampParsers: Array[TimestampParser]): Try[Unit] =
+    Try {
+      val parser = timestampParsers(column.getIndex)
+      pageBuilder.setTimestamp(column, parser.parse(value))
+    }
+
+  private[output] def writeJson(column: Column, value: String): Try[Unit] =
+    Try(pageBuilder.setJson(column, jsonParser.parse(value)))
+
+}

data/src/main/scala/org/embulk/parser/header_based_csv/tokenizer/TokenizerHelper.scala

@@ -0,0 +1,28 @@
+package org.embulk.parser.header_based_csv.tokenizer
+
+import org.embulk.standards.CsvTokenizer
+
+import scala.collection.mutable
+
+object TokenizerHelper {
+
+  implicit class RichCsvTokenizer(tokenizer: CsvTokenizer) {
+    def forEachRecord(f: Map[String, String] => Unit): Unit = {
+      val header: Seq[String] = if (tokenizer.nextRecord()) parseLine(tokenizer) else Seq.empty
+      while (tokenizer.nextRecord()) {
+        val values = parseLine(tokenizer)
+        val record = header.zip(values).toMap
+        f(record)
+      }
+    }
+  }
+
+  private def parseLine(tokenizer: CsvTokenizer): Seq[String] = {
+    val values = mutable.ListBuffer.empty[String]
+    while (tokenizer.hasNextColumn) {
+      values += tokenizer.nextColumn()
+    }
+    values.reverse.toList
+  }
+
+}

data/src/test/scala/org/embulk/parser/header_based_csv/TestHeaderBasedCsvParserPluginSpec.scala

@@ -0,0 +1,3 @@
+package org.embulk.parser.header_based_csv
+
+class TestHeaderBasedCsvParserPluginSpec

data/test.yml

@@ -0,0 +1,11 @@
+in:
+  type: file
+  path_prefix: csv
+  parser:
+    type: header_based_csv
+    columns:
+    - {name: a, type: long}
+    - {name: b, type: string}
+    - {name: c, type: double}
+out:
+  type: stdout

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: embulk-parser-header_based_csv
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- k_higuchi
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Parses Header Based Csv files read by other file input plugins.
+email:
+- k_higuchi@septeni-original.co.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".gitlab-ci.yml"
+- ".scalafmt.conf"
+- LICENSE.txt
+- README.md
+- build.gradle
+- classpath/commons-compress-1.10.jar
+- classpath/embulk-parser-header_based_csv-0.1.0.jar
+- classpath/embulk-standards-0.9.7.jar
+- classpath/scala-library-2.12.6.jar
+- csv/test_1.csv
+- csv/test_2.csv
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/guess/header_based_csv.rb
+- lib/embulk/parser/header_based_csv.rb
+- src/main/scala/helper/GoogleOptionalHelper.scala
+- src/main/scala/org/embulk/parser/header_based_csv/HeaderBasedCsvParserPlugin.scala
+- src/main/scala/org/embulk/parser/header_based_csv/exception/ColumnValueNotFoundException.scala
+- src/main/scala/org/embulk/parser/header_based_csv/exception/TooManyColumnValuesException.scala
+- src/main/scala/org/embulk/parser/header_based_csv/output/ColumnValueWriter.scala
+- src/main/scala/org/embulk/parser/header_based_csv/tokenizer/TokenizerHelper.scala
+- src/test/scala/org/embulk/parser/header_based_csv/TestHeaderBasedCsvParserPluginSpec.scala
+- test.yml
+homepage:
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
+specification_version: 4
+summary: Header Based Csv parser plugin for Embulk
+test_files: []