embulk-output-s3_per_record 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bc9e8ea415e7bb44a3c3c5395fcfdd40605a4885
-  data.tar.gz: ed4fa3983ab263f06c3c1293da4f818d894e4bd7
+  metadata.gz: cf8a267d6dac20c30dd45de82a04f3f47e670fdb
+  data.tar.gz: 9ebdc01acab6fb9da2e9b6311c01132f137ff7c6
 SHA512:
-  metadata.gz: 603e7b62bddb09d2246c1873b71da7e2d73819442822680337adda5453a373292f6b413ddcf9eef527e66d1931e1c049a4d9e3cc9d2297f1d4876b31cc506beb
-  data.tar.gz: 99d2923ecbf1638a21fd571b5d8883683168101abe9080d9debe8e6188cf07b8f8d0c19ea9884acae73754216b20fb69995f3cc8a9819ed5570be0c0c8cf024b
+  metadata.gz: 6b642a794a73d81ec10f3c9a530648124af38594e627c5e874cd9ab5f5ee02aa2e804f63a57717ba058f4a348f614059f17b998df64c967739ccd3275b80f8e4
+  data.tar.gz: 56ddc01a455e8c5a12cfaea19378e6b924e92eeb2f7d09266101f5f173e54598f09e93237df02cca44ce0fba0effe0e67c469892bfb326cf7b92188bb0435925

data/.gitignore

@@ -11,3 +11,5 @@ build/
 .classpath
 .project
 *.iml
+
+.envrc

data/README.md

@@ -3,6 +3,18 @@
 This plugin uploads a column's value to S3 as one S3 object per row.
 S3 object key can be composed of another column.
 
+## Breaking Changes from 0.3.x
+This plugin serialize data columns by itself.
+at present, Supported formats are `msgpack` and `json`.
+Because of it, config parameters are changed.
+
+- Rename `data_column` to `data_columns`, and change type from `string` to `array`
+- Add `mode`
+- Add `serializer`
+- Add `column_options`.
+
+Please update your configs.
+
 ## Overview
 
 * **Plugin type**: output
@@ -13,33 +25,61 @@ S3 object key can be composed of another column.
 
 - **bucket**: S3 bucket name.
 - **key**: S3 object key. `${column}` is replaced by the column's value.
-- **data_column**: Column for object's body.
+- **data_columns**: Columns for object's body.
+- **serializer**: Serializer format. Supported formats are `msgpack` and `json`. default is `msgpack`.
+- **mode**: Set mode. Supported modes are `multi_column` and `single_column`. default is `multi_column`.
 - **aws_access_key_id**: (optional) AWS access key id. If not given, [DefaultAWSCredentialsProviderChain](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) is used to get credentials.
 - **aws_secret_access_key**: (optional) AWS secret access key. Required if `aws_access_key_id` is given.
-- **base64**: (default false) If true, decode the value as Base64 before uploading.
 - **retry_limit**: (default 2) On connection errors,this plugin automatically retry up to this times.
+- **column_options**: Timestamp formatting option for columns.
 
 ## Example
 
+### multi_column mode
+
 ```yaml
 out:
   type: s3_per_record
   bucket: your-bucket-name
   key: "sample/${id}.txt"
-  data_column: payload
+  mode: multi_column
+  serializer: msgpack
+  data_columns: [id, payload]
 ```
 
 ```
-id | payload
+id | payload (json type)
 ------------
  1 | hello
  5 | world
 12 | embulk
 ```
 
-This generates `s3://your-bucket-name/sample/1.txt` with its content `hello`,
-`s3://your-bucket-name/sample/5.txt` with its content `world`, and so on.
+This generates `s3://your-bucket-name/sample/1.txt` with its content `{"id": 1, "payload": "hello"}` that is formatted with msgpack format,
+`s3://your-bucket-name/sample/5.txt` with its content `{"id": 5, "payload": "world"}`, and so on.
+
+### single_column mode
+
+```yaml
+out:
+  type: s3_per_record
+  bucket: your-bucket-name
+  key: "sample/${id}.txt"
+  mode: single_column
+  serializer: msgpack
+  data_columns: [payload]
+```
+
+```
+id | payload (json type)
+------------
+ 1 | hello
+ 5 | world
+12 | embulk
+```
 
+This generates `s3://your-bucket-name/sample/1.txt` with its content `"hello"` that is formatted with msgpack format,
+`s3://your-bucket-name/sample/5.txt` with its content `"world"`, and so on.
 
 ## Build
 

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.3.1"
+version = "0.4.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -22,6 +22,7 @@ dependencies {
     compile  "org.embulk:embulk-core:0.8.8"
     provided "org.embulk:embulk-core:0.8.8"
     compile  'com.amazonaws:aws-java-sdk-s3:1.10.71'
+    compile  'org.msgpack:jackson-dataformat-msgpack:0.8.3'
     testCompile "junit:junit:4.+"
 }
 

data/example/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_JOBS: 3

data/example/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org/'
+
+gem "embulk-parser-jsonl"

data/example/Gemfile.lock

@@ -0,0 +1,13 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    embulk-parser-jsonl (0.2.0)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  embulk-parser-jsonl
+
+BUNDLED WITH
+   1.10.6

data/example/example_array.json

@@ -0,0 +1 @@
+{"key": "test_key1", "data": [1,2,3]}

data/example/example_array.yml.liquid

@@ -0,0 +1,16 @@
+in:
+  type: file
+  path_prefix: "example_array.json"
+  parser:
+    type: jsonl
+    columns:
+      - {name: "key", type: "string"}
+      - {name: "data", type: "json"}
+
+out:
+  type: s3_per_record
+  data_columns: [data]
+  mode: single_column
+  bucket: {{ env.S3_BUCKET }}
+  key: "${key}"
+  serializer: msgpack

data/example/example_long.json

@@ -0,0 +1 @@
+{"key": "test_key1", "data": 1}

data/example/example_long.yml.liquid

@@ -0,0 +1,16 @@
+in:
+  type: file
+  path_prefix: "example_long.json"
+  parser:
+    type: jsonl
+    columns:
+      - {name: "key", type: "string"}
+      - {name: "data", type: "long"}
+
+out:
+  type: s3_per_record
+  data_columns: [data]
+  mode: single_column
+  bucket: {{ env.S3_BUCKET }}
+  key: "${key}"
+  serializer: msgpack

data/example/example_map.json

@@ -0,0 +1,2 @@
+{"key": "test_key1", "data": {"a": 1, "b": 2, "c": 3}}
+{"key": "test_key2", "data": ["foo", "bar", "buz"]}

data/example/example_map.yml.liquid

@@ -0,0 +1,16 @@
+in:
+  type: file
+  path_prefix: "example_map.json"
+  parser:
+    type: jsonl
+    columns:
+      - {name: "key", type: "string"}
+      - {name: "data", type: "json"}
+
+out:
+  type: s3_per_record
+  data_columns: [data]
+  mode: single_column
+  bucket: {{ env.S3_BUCKET }}
+  key: "${key}"
+  serializer: msgpack

data/example/example_map_json.yml.liquid

@@ -0,0 +1,16 @@
+in:
+  type: file
+  path_prefix: "example_map.json"
+  parser:
+    type: jsonl
+    columns:
+      - {name: "key", type: "string"}
+      - {name: "data", type: "json"}
+
+out:
+  type: s3_per_record
+  data_columns: [data]
+  mode: single_column
+  bucket: {{ env.S3_BUCKET }}
+  key: "${key}"
+  serializer: json

data/example/example_multicolumn.json

@@ -0,0 +1,3 @@
+{"key": "test_key_multi1", "data1": {"a": 1, "b": 2, "c": 3}, "data2": [1, 2], "data3": 42, "data4": "2016-10-01 10:21:00"}
+{"key": "test_key_multi2", "data1": {"a": 4, "b": 5, "c": 6}, "data2": [2, 4], "data3": 42, "data4": "2016-10-01 11:21:00"}
+{"key": "test_key_multi3", "data1": {"a": 7, "b": 8, "c": 9}, "data2": [4, 8], "data3": 42, "data4": "2016-10-01 12:21:00"}

data/example/example_multicolumn.yml.liquid

@@ -0,0 +1,23 @@
+in:
+  type: file
+  path_prefix: "example_multicolumn.json"
+  parser:
+    type: jsonl
+    columns:
+      - {name: "key", type: "string"}
+      - {name: "data1", type: "json"}
+      - {name: "data2", type: "json"}
+      - {name: "data3", type: "long"}
+      - {name: "data4", type: "timestamp", format: "%Y-%m-%d %H:%M:%S", timezone: "Asia/Tokyo"}
+
+out:
+  type: s3_per_record
+  data_columns:
+    - data1
+    - data2
+    - data3
+    - data4
+  mode: multi_column
+  bucket: {{ env.S3_BUCKET }}
+  key: "${key}"
+  serializer: msgpack

data/example/example_multicolumn_json.yml.liquid

@@ -0,0 +1,25 @@
+in:
+  type: file
+  path_prefix: "example_multicolumn.json"
+  parser:
+    type: jsonl
+    columns:
+      - {name: "key", type: "string"}
+      - {name: "data1", type: "json"}
+      - {name: "data2", type: "json"}
+      - {name: "data3", type: "long"}
+      - {name: "data4", type: "timestamp", format: "%Y-%m-%d %H:%M:%S", timezone: "Asia/Tokyo"}
+
+out:
+  type: s3_per_record
+  data_columns:
+    - data1
+    - data2
+    - data3
+    - data4
+  mode: multi_column
+  bucket: {{ env.S3_BUCKET }}
+  key: "${key}"
+  serializer: json
+  column_options:
+    data4: {format: "%Y/%m/%d %H:%M:%S %z", timezone: "Asia/Tokyo"}

data/example/example_string.json

@@ -0,0 +1,2 @@
+{"key": "test_key1", "data": "foo"}
+{"key": "test_key2", "data": "\"aa\\aa\""}

data/example/example_string.yml.liquid

@@ -0,0 +1,16 @@
+in:
+  type: file
+  path_prefix: "example_string.json"
+  parser:
+    type: jsonl
+    columns:
+      - {name: "key", type: "string"}
+      - {name: "data", type: "string"}
+
+out:
+  type: s3_per_record
+  data_columns: [data]
+  mode: single_column
+  bucket: {{ env.S3_BUCKET }}
+  key: "${key}"
+  serializer: msgpack

data/example/example_string_json.yml.liquid

@@ -0,0 +1,16 @@
+in:
+  type: file
+  path_prefix: "example_string.json"
+  parser:
+    type: jsonl
+    columns:
+      - {name: "key", type: "string"}
+      - {name: "data", type: "string"}
+
+out:
+  type: s3_per_record
+  data_columns: [data]
+  mode: single_column
+  bucket: {{ env.S3_BUCKET }}
+  key: "${key}"
+  serializer: json

data/example/example_timestamp.json

@@ -0,0 +1 @@
+{"key": "test_key1", "data": "2016-10-01 10:21:00"}

data/example/example_timestamp.yml.liquid

@@ -0,0 +1,18 @@
+in:
+  type: file
+  path_prefix: "example_timestamp.json"
+  parser:
+    type: jsonl
+    columns:
+      - {name: "key", type: "string"}
+      - {name: "data", type: "timestamp", format: "%Y-%m-%d %H:%M:%S", timezone: "Asia/Tokyo"}
+
+out:
+  type: s3_per_record
+  data_columns: [data]
+  mode: single_column
+  bucket: {{ env.S3_BUCKET }}
+  key: "${key}"
+  serializer: msgpack
+  column_options:
+    data: {format: "%Y/%m/%d %H:%M:%S %z"}

data/example/example_timestamp_json.yml.liquid

@@ -0,0 +1,16 @@
+in:
+  type: file
+  path_prefix: "example_timestamp.json"
+  parser:
+    type: jsonl
+    columns:
+      - {name: "key", type: "string"}
+      - {name: "data", type: "timestamp", format: "%Y-%m-%d %H:%M:%S", timezone: "Asia/Tokyo"}
+
+out:
+  type: s3_per_record
+  data_columns: [data]
+  mode: single_column
+  bucket: {{ env.S3_BUCKET }}
+  key: "${key}"
+  serializer: json

data/gradlew.bat

@@ -1,90 +1,90 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windowz variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java

@@ -1,23 +1,28 @@
 package org.embulk.output.s3_per_record;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-import javax.validation.constraints.NotNull;
-
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.Upload;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
+import org.embulk.output.s3_per_record.visitor.JsonMultiColumnVisitor;
+import org.embulk.output.s3_per_record.visitor.JsonSingleColumnVisitor;
+import org.embulk.output.s3_per_record.visitor.MessagePackMultiColumnVisitor;
+import org.embulk.output.s3_per_record.visitor.MessagePackSingleColumnVisitor;
+import org.embulk.output.s3_per_record.visitor.S3PerRecordOutputColumnVisitor;
 import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
@@ -25,17 +30,20 @@ import org.embulk.spi.Page;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.transfer.TransferManager;
-import com.amazonaws.services.s3.transfer.Upload;
-import com.amazonaws.util.Base64;
-import com.google.common.base.Optional;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.util.Timestamps;
 import org.slf4j.Logger;
 
+import javax.validation.constraints.NotNull;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
 
 public class S3PerRecordOutputPlugin
         implements OutputPlugin
@@ -50,7 +58,7 @@ public class S3PerRecordOutputPlugin
     private static long startTime = System.currentTimeMillis();
 
     public interface PluginTask
-            extends Task
+            extends Task, TimestampFormatter.Task
     {
         // S3 bucket name.
         @Config("bucket")
@@ -61,8 +69,9 @@ public class S3PerRecordOutputPlugin
         String getKey();
 
         // Column name.
-        @Config("data_column")
-        String getDataColumn();
+        @Config("data_columns")
+        @ConfigDefault("[]")
+        List<String> getDataColumns();
 
         // AWS access key id.
         @Config("aws_access_key_id")
@@ -74,17 +83,28 @@ public class S3PerRecordOutputPlugin
         @ConfigDefault("null")
         Optional<String> getAwsSecretAccessKey();
 
-        // Enable Base64 decoding
-        @Config("base64")
-        @ConfigDefault("false")
-        boolean getBase64();
+        @Config("serializer")
+        @ConfigDefault("msgpack")
+        Serializer getSerializer();
+
+        @Config("mode")
+        @ConfigDefault("multi_column")
+        Mode getMode();
 
         // Set retry limit. Default is 2.
         @Config("retry_limit")
         @ConfigDefault("2")
         Integer getRetryLimit();
+
+        @Config("column_options")
+        @ConfigDefault("{}")
+        Map<String, TimestampColumnOption> getColumnOptions();
     }
 
+    public interface TimestampColumnOption
+            extends Task, TimestampFormatter.TimestampColumnOption
+    { }
+
     @Override
     public ConfigDiff transaction(ConfigSource config,
             Schema schema, int taskCount,
@@ -124,18 +144,22 @@ public class S3PerRecordOutputPlugin
         private PageReader pageReader;
         private final String bucket;
         private final List<KeyPart> keyPattern;
-        private final Column dataColumn;
+        private final List<String> dataColumns;
         private final Schema schema;
-        private final boolean decodeBase64;
         private final int retryLimit;
+        private final Serializer serializer;
+        private final Mode mode;
+        private final TimestampFormatter[] timestampFormatters;
 
         public S3PerRecordPageOutput(PluginTask task, Schema schema) {
             this.schema = schema;
             bucket = task.getBucket();
             keyPattern = makeKeyPattern(task.getKey());
-            dataColumn = schema.lookupColumn(task.getDataColumn());
-            decodeBase64 = task.getBase64();
+            dataColumns = task.getDataColumns();
             retryLimit = task.getRetryLimit();
+            serializer = task.getSerializer();
+            mode = task.getMode();
+            timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
 
             AWSCredentials credentials;
             if (task.getAwsAccessKeyId().isPresent() && task.getAwsSecretAccessKey().isPresent()) {
@@ -176,15 +200,9 @@ public class S3PerRecordOutputPlugin
             pageReader.setPage(page);
 
             while (pageReader.nextRecord()) {
-                String key = buildKey(pageReader);
-
-                String payload = pageReader.getString(dataColumn);
-                byte[] payloadBytes;
-                if (decodeBase64) {
-                    payloadBytes = Base64.decode(payload);
-                } else {
-                    payloadBytes = payload.getBytes(StandardCharsets.UTF_8);
-                }
+                final String key = buildKey(pageReader);
+                final byte[] payloadBytes = serializer.serialize(mode, pageReader, schema, dataColumns, timestampFormatters);
+
                 ObjectMetadata metadata = new ObjectMetadata();
                 metadata.setContentLength(payloadBytes.length);
 
@@ -286,4 +304,96 @@ public class S3PerRecordOutputPlugin
             return reader.getString(column);
         }
     }
+
+    public enum Serializer {
+        MSGPACK {
+            @Override
+            public byte[] serialize(Mode mode, PageReader reader, Schema schema, List<String> dataColumns, TimestampFormatter[] timestampFormatters) {
+                S3PerRecordOutputColumnVisitor visitor;
+                switch(mode) {
+                    case SINGLE_COLUMN:
+                        visitor = new MessagePackSingleColumnVisitor(reader, timestampFormatters);
+                        schema.lookupColumn(dataColumns.get(0)).visit(visitor);
+                        break;
+                    case MULTI_COLUMN:
+                        visitor = new MessagePackMultiColumnVisitor(reader, timestampFormatters);
+                        for (String columnName : dataColumns) {
+                            schema.lookupColumn(columnName).visit(visitor);
+                        }
+                        break;
+                    default:
+                        throw new RuntimeException("never reach here");
+                }
+                return visitor.getByteArray();
+            }
+        },
+        JSON {
+            @Override
+            public byte[] serialize(Mode mode, PageReader reader, Schema schema, List<String> dataColumns, TimestampFormatter[] timestampFormatters) {
+                S3PerRecordOutputColumnVisitor visitor;
+                switch(mode) {
+                    case SINGLE_COLUMN:
+                        visitor = new JsonSingleColumnVisitor(reader, timestampFormatters);
+                        schema.lookupColumn(dataColumns.get(0)).visit(visitor);
+                        break;
+                    case MULTI_COLUMN:
+                        visitor = new JsonMultiColumnVisitor(reader, timestampFormatters);
+                        for (String columnName : dataColumns) {
+                            schema.lookupColumn(columnName).visit(visitor);
+                        }
+                        break;
+                    default:
+                        throw new RuntimeException("never reach here");
+                }
+                return visitor.getByteArray();
+            }
+        };
+
+        public abstract byte[] serialize(Mode mode, PageReader reader, Schema schema, List<String> dataColumns, TimestampFormatter[] timestampFormatters);
+
+        @JsonValue
+        @Override
+        public String toString()
+        {
+            return name().toLowerCase(Locale.ENGLISH);
+        }
+
+        @JsonCreator
+        public static Serializer fromString(String name)
+        {
+            switch(name) {
+                case "msgpack":
+                    return MSGPACK;
+                case "json":
+                    return JSON;
+                default:
+                    throw new ConfigException(String.format("Unknown format '%s'. Supported formats are msgpack only", name));
+            }
+        }
+    }
+
+    public enum Mode {
+        SINGLE_COLUMN,
+        MULTI_COLUMN;
+
+        @JsonValue
+        @Override
+        public String toString()
+        {
+            return name().toLowerCase(Locale.ENGLISH);
+        }
+
+        @JsonCreator
+        public static Mode fromString(String name)
+        {
+            switch(name) {
+                case "single_column":
+                    return SINGLE_COLUMN;
+                case "multi_column":
+                    return MULTI_COLUMN;
+                default:
+                    throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are single_column, multi_column", name));
+            }
+        }
+    }
 }

data/src/main/java/org/embulk/output/s3_per_record/visitor/JsonMultiColumnVisitor.java

@@ -0,0 +1,73 @@
+package org.embulk.output.s3_per_record.visitor;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.time.TimestampFormatter;
+import org.msgpack.core.MessageBufferPacker;
+import org.msgpack.core.MessagePack;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class JsonMultiColumnVisitor implements S3PerRecordOutputColumnVisitor {
+    final PageReader reader;
+    final TimestampFormatter[] timestampFormatters;
+    public final ValueFactory.MapBuilder builder;
+
+    public JsonMultiColumnVisitor(PageReader reader, TimestampFormatter[] timestampFormatters) {
+        this.reader = reader;
+        this.timestampFormatters = timestampFormatters;
+        this.builder = new ValueFactory.MapBuilder();
+    }
+
+    public byte[] getByteArray() {
+        Value value = builder.build();
+        String json = value.toJson();
+        return json.getBytes(StandardCharsets.UTF_8);
+    }
+
+    @Override
+    public void booleanColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        Value value = ValueFactory.newBoolean(reader.getBoolean(column));
+        builder.put(columnName, value);
+    }
+
+    @Override
+    public void longColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        Value value = ValueFactory.newInteger(reader.getLong(column));
+        builder.put(columnName, value);
+    }
+
+    @Override
+    public void doubleColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        Value value = ValueFactory.newFloat(reader.getDouble(column));
+        builder.put(columnName, value);
+    }
+
+    @Override
+    public void stringColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        Value value = ValueFactory.newString(reader.getString(column));
+        builder.put(columnName, value);
+    }
+
+    @Override
+    public void timestampColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        TimestampFormatter formatter = timestampFormatters[column.getIndex()];
+        Value value = ValueFactory.newString(formatter.format(reader.getTimestamp(column)));
+        builder.put(columnName, value);
+    }
+
+    @Override
+    public void jsonColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        Value value = reader.getJson(column);
+        builder.put(columnName, value);
+    }
+}

data/src/main/java/org/embulk/output/s3_per_record/visitor/JsonSingleColumnVisitor.java

@@ -0,0 +1,63 @@
+package org.embulk.output.s3_per_record.visitor;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import java.nio.charset.StandardCharsets;
+
+public class JsonSingleColumnVisitor implements S3PerRecordOutputColumnVisitor {
+    final PageReader reader;
+    final TimestampFormatter[] timestampFormatters;
+    final StringBuilder sb;
+
+    public JsonSingleColumnVisitor(PageReader reader, TimestampFormatter[] timestampFormatters) {
+        this.reader = reader;
+        this.timestampFormatters = timestampFormatters;
+        this.sb = new StringBuilder();
+    }
+
+    public byte[] getByteArray() {
+        return sb.toString().getBytes(StandardCharsets.UTF_8);
+    }
+
+    @Override
+    public void booleanColumn(Column column) {
+        boolean value = reader.getBoolean(column);
+        sb.append(value);
+    }
+
+    @Override
+    public void longColumn(Column column) {
+        long value = reader.getLong(column);
+        sb.append(value);
+    }
+
+    @Override
+    public void doubleColumn(Column column) {
+        double value = reader.getDouble(column);
+        sb.append(value);
+    }
+
+    @Override
+    public void stringColumn(Column column) {
+        String value = reader.getString(column);
+        sb.append(ValueFactory.newString(value).toJson());
+    }
+
+    @Override
+    public void timestampColumn(Column column) {
+        Timestamp value = reader.getTimestamp(column);
+        TimestampFormatter formatter = timestampFormatters[column.getIndex()];
+        sb.append(ValueFactory.newString(formatter.format(value)).toJson());
+    }
+
+    @Override
+    public void jsonColumn(Column column) {
+        Value value = reader.getJson(column);
+        sb.append(value.toJson());
+    }
+}

data/src/main/java/org/embulk/output/s3_per_record/visitor/MessagePackMultiColumnVisitor.java

@@ -0,0 +1,77 @@
+package org.embulk.output.s3_per_record.visitor;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.time.TimestampFormatter;
+import org.msgpack.core.MessageBufferPacker;
+import org.msgpack.core.MessagePack;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import java.io.IOException;
+
+public class MessagePackMultiColumnVisitor implements S3PerRecordOutputColumnVisitor {
+    final PageReader reader;
+    final TimestampFormatter[] timestampFormatters;
+    public final ValueFactory.MapBuilder builder;
+
+    public MessagePackMultiColumnVisitor(PageReader reader, TimestampFormatter[] timestampFormatters) {
+        this.reader = reader;
+        this.timestampFormatters = timestampFormatters;
+        this.builder = new ValueFactory.MapBuilder();
+    }
+
+    public byte[] getByteArray() {
+        Value value = builder.build();
+        MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();
+        try {
+            value.writeTo(packer);
+            return packer.toByteArray();
+        } catch (IOException e) {
+            throw new RuntimeException("cannot write to msgpack");
+        }
+    }
+
+    @Override
+    public void booleanColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        Value value = ValueFactory.newBoolean(reader.getBoolean(column));
+        builder.put(columnName, value);
+    }
+
+    @Override
+    public void longColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        Value value = ValueFactory.newInteger(reader.getLong(column));
+        builder.put(columnName, value);
+    }
+
+    @Override
+    public void doubleColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        Value value = ValueFactory.newFloat(reader.getDouble(column));
+        builder.put(columnName, value);
+    }
+
+    @Override
+    public void stringColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        Value value = ValueFactory.newString(reader.getString(column));
+        builder.put(columnName, value);
+    }
+
+    @Override
+    public void timestampColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        TimestampFormatter formatter = timestampFormatters[column.getIndex()];
+        Value value = ValueFactory.newString(formatter.format(reader.getTimestamp(column)));
+        builder.put(columnName, value);
+    }
+
+    @Override
+    public void jsonColumn(Column column) {
+        Value columnName = ValueFactory.newString(column.getName());
+        Value value = reader.getJson(column);
+        builder.put(columnName, value);
+    }
+}

data/src/main/java/org/embulk/output/s3_per_record/visitor/MessagePackSingleColumnVisitor.java

@@ -0,0 +1,88 @@
+package org.embulk.output.s3_per_record.visitor;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+import org.msgpack.core.MessageBufferPacker;
+import org.msgpack.core.MessagePack;
+import org.msgpack.value.Value;
+
+import java.io.IOException;
+
+public class MessagePackSingleColumnVisitor implements S3PerRecordOutputColumnVisitor {
+    final PageReader reader;
+    final TimestampFormatter[] timestampFormatters;
+    final MessageBufferPacker packer;
+
+    public MessagePackSingleColumnVisitor(PageReader reader, TimestampFormatter[] timestampFormatters) {
+        this.reader = reader;
+        this.timestampFormatters = timestampFormatters;
+        this.packer = MessagePack.newDefaultBufferPacker();
+    }
+
+    public byte[] getByteArray() {
+        return packer.toByteArray();
+    }
+
+    @Override
+    public void booleanColumn(Column column) {
+        boolean value = reader.getBoolean(column);
+        try {
+            packer.packBoolean(value);
+        } catch (IOException e) {
+            throw new RuntimeException("cannot write to msgpack");
+        }
+    }
+
+    @Override
+    public void longColumn(Column column) {
+        long value = reader.getLong(column);
+        try {
+            packer.packLong(value);
+        } catch (IOException e) {
+            throw new RuntimeException("cannot write to msgpack");
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column column) {
+        double value = reader.getDouble(column);
+        try {
+            packer.packDouble(value);
+        } catch (IOException e) {
+            throw new RuntimeException("cannot write to msgpack");
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column) {
+        String value = reader.getString(column);
+        try {
+            packer.packString(value);
+        } catch (IOException e) {
+            throw new RuntimeException("cannot write to msgpack");
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column column) {
+        Timestamp value = reader.getTimestamp(column);
+        TimestampFormatter formatter = timestampFormatters[column.getIndex()];
+        try {
+            packer.packString(formatter.format(value));
+        } catch (IOException e) {
+            throw new RuntimeException("cannot write to msgpack");
+        }
+    }
+
+    @Override
+    public void jsonColumn(Column column) {
+        Value value = reader.getJson(column);
+        try {
+            value.writeTo(packer);
+        } catch (IOException e) {
+            throw new RuntimeException("cannot write to msgpack");
+        }
+    }
+}

data/src/main/java/org/embulk/output/s3_per_record/visitor/S3PerRecordOutputColumnVisitor.java

@@ -0,0 +1,7 @@
+package org.embulk.output.s3_per_record.visitor;
+
+import org.embulk.spi.ColumnVisitor;
+
+public interface S3PerRecordOutputColumnVisitor extends ColumnVisitor {
+    public byte[] getByteArray();
+}

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-s3_per_record
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - tomykaira
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-02 00:00:00.000000000 Z
+date: 2016-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  prerelease: false
+  type: :development
 description: Upload a column's value to S3 as one S3 object per row.
 email:
 - tomykaira@gmail.com
@@ -51,22 +51,50 @@ files:
 - build.gradle
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
+- example/.bundle/config
+- example/Gemfile
+- example/Gemfile.lock
+- example/example_array.json
+- example/example_array.yml.liquid
+- example/example_long.json
+- example/example_long.yml.liquid
+- example/example_map.json
+- example/example_map.yml.liquid
+- example/example_map_json.yml.liquid
+- example/example_multicolumn.json
+- example/example_multicolumn.yml.liquid
+- example/example_multicolumn_json.yml.liquid
+- example/example_string.json
+- example/example_string.yml.liquid
+- example/example_string_json.yml.liquid
+- example/example_timestamp.json
+- example/example_timestamp.yml.liquid
+- example/example_timestamp_json.yml.liquid
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
 - gradlew.bat
 - lib/embulk/output/s3_per_record.rb
 - src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java
+- src/main/java/org/embulk/output/s3_per_record/visitor/JsonMultiColumnVisitor.java
+- src/main/java/org/embulk/output/s3_per_record/visitor/JsonSingleColumnVisitor.java
+- src/main/java/org/embulk/output/s3_per_record/visitor/MessagePackMultiColumnVisitor.java
+- src/main/java/org/embulk/output/s3_per_record/visitor/MessagePackSingleColumnVisitor.java
+- src/main/java/org/embulk/output/s3_per_record/visitor/S3PerRecordOutputColumnVisitor.java
 - src/test/java/org/embulk/output/s3_per_record/TestS3PerRecordOutputPlugin.java
-- classpath/jackson-dataformat-cbor-2.5.3.jar
-- classpath/httpclient-4.3.6.jar
-- classpath/commons-logging-1.1.3.jar
-- classpath/commons-codec-1.6.jar
-- classpath/aws-java-sdk-s3-1.10.71.jar
 - classpath/aws-java-sdk-core-1.10.71.jar
-- classpath/embulk-output-s3_per_record-0.3.1.jar
-- classpath/httpcore-4.3.3.jar
 - classpath/aws-java-sdk-kms-1.10.71.jar
+- classpath/aws-java-sdk-s3-1.10.71.jar
+- classpath/commons-codec-1.6.jar
+- classpath/commons-logging-1.1.3.jar
+- classpath/embulk-output-s3_per_record-0.4.0.jar
+- classpath/httpclient-4.3.6.jar
+- classpath/httpcore-4.3.3.jar
+- classpath/jackson-annotations-2.7.0.jar
+- classpath/jackson-core-2.7.1.jar
+- classpath/jackson-databind-2.7.1.jar
+- classpath/jackson-dataformat-cbor-2.5.3.jar
+- classpath/jackson-dataformat-msgpack-0.8.3.jar
 homepage: https://github.com/tomykaira/embulk-output-s3_per_record
 licenses:
 - MIT